JUnit version 4.13.2
.20:03:15.506 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783540995499
20:03:15.566 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
20:03:15.626 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:03:15.639 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskZLZTjO/test-tmp
20:03:15.644 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:03:15.647 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskZLZTjO/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:03:15.908 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

20:03:16.093 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
20:03:16.094 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

20:03:16.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:16.100025  6275 mini_oidc.cc:150] Starting JWKS server
20:03:16.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:16.104377  6275 webserver.cc:533] Webserver started at http://127.0.0.1:35631/ using document root <none> and password file <none>
20:03:16.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:16.105055  6275 mini_oidc.cc:186] Starting OIDC Discovery server
20:03:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:16.107582  6275 webserver.cc:533] Webserver started at http://127.0.0.1:44835/ using document root <none> and password file <none>
20:03:16.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T20:03:16Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:03:16.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T20:03:16Z Disabled control of system clock
20:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:16.176699  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:16.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:16.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/wal
20:03:16.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/data
20:03:16.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:16.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:16.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:16.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:16.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:16.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:16.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/logs
20:03:16.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/data/info.pb
20:03:16.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:16.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:16.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:16.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:16.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:16.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:16.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:16.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:16.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:16.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:16.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:16.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:16.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.254:43103
20:03:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.254
20:03:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35631/jwks.json with env {}
20:03:16.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.491668  6287 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:16.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.492257  6287 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:16.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.492799  6287 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:16.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.527316  6287 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:16.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.527631  6287 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:16.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.527879  6287 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.528126  6287 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.528335  6287 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.528477  6287 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35631/jwks.json
20:03:16.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:16.568101  6287 master_runner.cc:387] Master server non-default flags:
20:03:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/data
20:03:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/wal
20:03:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.254:43103
20:03:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35631/jwks.json
20:03:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/data/info.pb
20:03:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.254
20:03:16.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:16.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:16.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:16.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:16.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:16.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:16.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/logs
20:03:16.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:16.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:16.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:16.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:16.569588  6287 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:16.571791  6287 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:16.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.585121  6293 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:17.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:16.586884  6294 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:17.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:17.843860  6296 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:17.847128  6295 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1255 milliseconds
20:03:17.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:17.847710  6287 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.262s	user 0.385s	sys 0.867s
20:03:17.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:17.848098  6287 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.263s	user 0.385s	sys 0.867s
20:03:17.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.848415  6287 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.850086  6287 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.853506  6287 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.855024  6287 hybrid_clock.cc:648] HybridClock initialized: now 1783540997854967 us; error 52 us; skew 500 ppm
20:03:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.856290  6287 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:17.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.866060  6287 webserver.cc:533] Webserver started at http://127.6.32.254:36611/ using document root <none> and password file <none>
20:03:17.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.867746  6287 fs_manager.cc:362] Metadata directory not provided
20:03:17.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.868119  6287 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.868813  6287 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:17.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.876470  6287 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/data/instance:
20:03:17.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3dcc02af18cc424e80c917ff88a021ad"
20:03:17.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:17 on dist-test-slave-xfg5"
20:03:17.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.878351  6287 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/wal/instance:
20:03:17.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3dcc02af18cc424e80c917ff88a021ad"
20:03:17.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:17 on dist-test-slave-xfg5"
20:03:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.889782  6287 fs_manager.cc:696] Time spent creating directory manager: real 0.010s	user 0.007s	sys 0.003s
20:03:17.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.898893  6303 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.900503  6287 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.001s
20:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.901000  6287 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/wal
20:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3dcc02af18cc424e80c917ff88a021ad"
20:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:17 on dist-test-slave-xfg5"
20:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.902062  6287 fs_report.cc:389] FS layout report
20:03:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/wal
20:03:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/wal
20:03:17.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/data/data
20:03:17.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:17.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:17.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:17.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:17.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:17.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:17.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:17.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:17.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:17.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.994536  6287 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:17.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.996407  6287 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:17.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:17.996872  6287 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.076723  6287 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.254:43103
20:03:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.076807  6355 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.254:43103 every 8 connection(s)
20:03:18.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.080228  6287 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/data/info.pb
20:03:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.083309  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 6287
20:03:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.083868  6275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-0/wal/instance
20:03:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.088006  6356 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.089064  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/wal
20:03:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/data
20:03:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/logs
20:03:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/data/info.pb
20:03:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:18.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:18.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:18.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:18.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.253:39891
20:03:18.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.253
20:03:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:18.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35631/jwks.json with env {}
20:03:18.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.104045  6356 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } has no permanent_uuid. Determining permanent_uuid...
20:03:18.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.131462  6356 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } has no permanent_uuid. Determining permanent_uuid...
20:03:18.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.134290  6304 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.253:39891: connect: Connection refused (error 111)
20:03:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.139257  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.253:39891: Network error: Client connection negotiation failed: client connection to 127.6.32.253:39891: connect: Connection refused (error 111)
20:03:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.188941  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } attempt: 1
20:03:18.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.195393  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.253:39891: Network error: Client connection negotiation failed: client connection to 127.6.32.253:39891: connect: Connection refused (error 111)
20:03:18.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.264005  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } attempt: 2
20:03:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.268906  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.253:39891: Network error: Client connection negotiation failed: client connection to 127.6.32.253:39891: connect: Connection refused (error 111)
20:03:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.360428  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } attempt: 3
20:03:18.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.365216  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.253:39891: Network error: Client connection negotiation failed: client connection to 127.6.32.253:39891: connect: Connection refused (error 111)
20:03:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.418840  6358 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.419418  6358 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.419865  6358 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:18.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.455704  6358 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:18.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.456029  6358 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:18.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.456229  6358 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.456419  6358 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.456599  6358 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.456748  6358 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35631/jwks.json
20:03:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.507933  6358 master_runner.cc:387] Master server non-default flags:
20:03:18.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:18.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:18.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/data
20:03:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/wal
20:03:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.253:39891
20:03:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:18.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:18.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35631/jwks.json
20:03:18.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:18.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:18.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/data/info.pb
20:03:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.253
20:03:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:18.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:18.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:18.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/logs
20:03:18.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:18.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:18.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:18.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:18.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:18.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.508785  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } attempt: 4
20:03:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.510159  6358 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.512456  6358 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.514704  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.253:39891: Network error: Client connection negotiation failed: client connection to 127.6.32.253:39891: connect: Connection refused (error 111)
20:03:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.527156  6367 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:18.814216  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } attempt: 5
20:03:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.825268  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.253:39891: Network error: Client connection negotiation failed: client connection to 127.6.32.253:39891: connect: Connection refused (error 111)
20:03:19.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.373325  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } attempt: 6
20:03:19.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:19.408031  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.253:39891: Network error: Client connection negotiation failed: client connection to 127.6.32.253:39891: connect: Connection refused (error 111)
20:03:19.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:18.529006  6368 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:19.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:19.844607  6370 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:19.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:19.844758  6369 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1316 milliseconds
20:03:19.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:19.848038  6358 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.321s	user 0.408s	sys 0.904s
20:03:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:19.848668  6358 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.322s	user 0.408s	sys 0.904s
20:03:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.848914  6358 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:19.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.855995  6358 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:19.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.860178  6358 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:19.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.861653  6358 hybrid_clock.cc:648] HybridClock initialized: now 1783540999861612 us; error 57 us; skew 500 ppm
20:03:19.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.862773  6358 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.871456  6358 webserver.cc:533] Webserver started at http://127.6.32.253:45589/ using document root <none> and password file <none>
20:03:19.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.872776  6358 fs_manager.cc:362] Metadata directory not provided
20:03:19.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.873054  6358 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:19.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.873711  6358 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.878432  6358 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/data/instance:
20:03:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73a242ff54b84b15a1f7b1d5d9029e67"
20:03:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:19 on dist-test-slave-xfg5"
20:03:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.880057  6358 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/wal/instance:
20:03:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73a242ff54b84b15a1f7b1d5d9029e67"
20:03:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:19 on dist-test-slave-xfg5"
20:03:19.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.889312  6358 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.004s	sys 0.003s
20:03:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.896469  6379 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:19.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.897936  6358 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.002s
20:03:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.898465  6358 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/wal
20:03:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73a242ff54b84b15a1f7b1d5d9029e67"
20:03:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:19 on dist-test-slave-xfg5"
20:03:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.898967  6358 fs_report.cc:389] FS layout report
20:03:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/wal
20:03:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/wal
20:03:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/data/data
20:03:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:19.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:19.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:19.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.974560  6358 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.977088  6358 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:19.977826  6358 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:20.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.066855  6358 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.253:39891
20:03:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.066907  6430 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.253:39891 every 8 connection(s)
20:03:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.070151  6358 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/data/info.pb
20:03:20.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.078447  6431 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:20.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.079860  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 6358
20:03:20.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.080302  6275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-1/wal/instance
20:03:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.082561  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/wal
20:03:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/data
20:03:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/logs
20:03:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/data/info.pb
20:03:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:20.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.252:42851
20:03:20.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.252
20:03:20.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:20.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:20.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:20.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:20.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:20.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:20.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:20.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35631/jwks.json with env {}
20:03:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.098645  6431 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } has no permanent_uuid. Determining permanent_uuid...
20:03:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.134780  6431 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } has no permanent_uuid. Determining permanent_uuid...
20:03:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.152850  6431 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } has no permanent_uuid. Determining permanent_uuid...
20:03:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.156626  6383 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.162278  6431 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.211906  6431 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 1
20:03:20.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.217962  6431 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.286443  6431 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 2
20:03:20.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.291695  6431 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.383198  6431 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 3
20:03:20.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.387997  6431 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.434698  6433 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:20.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.435335  6433 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:20.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.435802  6433 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.468717  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } attempt: 7
20:03:20.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.469705  6433 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:20.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.470137  6433 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:20.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.470505  6433 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:20.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.471369  6433 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:20.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.472210  6433 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:20.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.472765  6433 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35631/jwks.json
20:03:20.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.482642  6356 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } has no permanent_uuid. Determining permanent_uuid...
20:03:20.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.489540  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.513631  6433 master_runner.cc:387] Master server non-default flags:
20:03:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/data
20:03:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/wal
20:03:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:20.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.252:42851
20:03:20.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:20.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:20.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:20.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35631/jwks.json
20:03:20.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:20.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:20.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/data/info.pb
20:03:20.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.252
20:03:20.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:20.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:20.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/logs
20:03:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.515110  6433 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.516873  6433 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.529996  6445 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:20.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.531800  6431 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 4
20:03:20.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.540637  6431 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.548043  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 1
20:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.553047  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.634582  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 2
20:03:20.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.640937  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.726487  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 3
20:03:20.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.732221  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.840271  6431 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 5
20:03:20.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.845142  6431 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:20.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:20.872907  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 4
20:03:20.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.880124  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:21.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.163573  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 5
20:03:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:21.169054  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:21.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.392860  6431 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 6
20:03:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:21.401201  6431 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.721482  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 6
20:03:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:21.733455  6356 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.32.252:42851: Network error: Client connection negotiation failed: client connection to 127.6.32.252:42851: connect: Connection refused (error 111)
20:03:21.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:20.531661  6446 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:21.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:21.773171  6447 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:03:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:21.774521  6448 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.774653  6433 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:21.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.778409  6433 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:21.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.781725  6433 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:21.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.783452  6433 hybrid_clock.cc:648] HybridClock initialized: now 1783541001783411 us; error 44 us; skew 500 ppm
20:03:21.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.784543  6433 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.791785  6433 webserver.cc:533] Webserver started at http://127.6.32.252:33911/ using document root <none> and password file <none>
20:03:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.792765  6433 fs_manager.cc:362] Metadata directory not provided
20:03:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.792973  6433 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.793368  6433 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:21.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.798326  6433 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/data/instance:
20:03:21.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0b84cba6b4f4afa8fd74c677579c439"
20:03:21.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:21 on dist-test-slave-xfg5"
20:03:21.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.799567  6433 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/wal/instance:
20:03:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0b84cba6b4f4afa8fd74c677579c439"
20:03:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:21 on dist-test-slave-xfg5"
20:03:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.808627  6433 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.008s	sys 0.000s
20:03:21.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.815385  6459 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:21.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.816716  6433 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.004s
20:03:21.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.817098  6433 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/wal
20:03:21.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0b84cba6b4f4afa8fd74c677579c439"
20:03:21.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:21 on dist-test-slave-xfg5"
20:03:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.817554  6433 fs_report.cc:389] FS layout report
20:03:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/wal
20:03:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/wal
20:03:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/data/data
20:03:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.872697  6433 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:21.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.874473  6433 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:21.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.875102  6433 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.962951  6433 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.252:42851
20:03:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.963019  6510 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.252:42851 every 8 connection(s)
20:03:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.966126  6433 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/data/info.pb
20:03:21.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.971681  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 6433
20:03:21.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.972182  6275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/master-2/wal/instance
20:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.973137  6511 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:21.990511  6511 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } has no permanent_uuid. Determining permanent_uuid...
20:03:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.029667  6511 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } has no permanent_uuid. Determining permanent_uuid...
20:03:22.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.041093  6511 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } has no permanent_uuid. Determining permanent_uuid...
20:03:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.059592  6511 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439: Bootstrap starting.
20:03:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.068435  6511 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439: Neither blocks nor log segments found. Creating new log.
20:03:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.072156  6511 log.cc:826] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439: Log is configured to *not* fsync() on all Append() calls
20:03:22.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.088354  6511 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439: No bootstrap required, opened a new log
20:03:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.110770  6511 raft_consensus.cc:359] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } }
20:03:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.111431  6511 raft_consensus.cc:385] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.111692  6511 raft_consensus.cc:740] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b84cba6b4f4afa8fd74c677579c439, State: Initialized, Role: FOLLOWER
20:03:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.112641  6511 consensus_queue.cc:260] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } }
20:03:22.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.115374  6520 sys_catalog.cc:455] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } } }
20:03:22.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.115957  6520 sys_catalog.cc:458] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [sys.catalog]: This master's current role is: FOLLOWER
20:03:22.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.116819  6511 sys_catalog.cc:565] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [sys.catalog]: configured and running, proceeding with master startup.
20:03:22.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:22.131202  6465 tablet.cc:2406] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:22.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:22.146770  6531 catalog_manager.cc:1594] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:22.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:22.147177  6531 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.142455  6433 master_runner.cc:187] Error getting master registration for 127.6.32.254:43103: OK, instance_id { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" instance_seqno: 1783540998054328 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.166953  6433 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:03:22.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.214941  6520 raft_consensus.cc:493] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:22.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.215498  6520 raft_consensus.cc:515] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } }
20:03:22.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.221462  6520 leader_election.cc:290] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dcc02af18cc424e80c917ff88a021ad (127.6.32.254:43103), 73a242ff54b84b15a1f7b1d5d9029e67 (127.6.32.253:39891)
20:03:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.461854  6431 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 7
20:03:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.487979  6431 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67: Bootstrap starting.
20:03:22.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.496640  6431 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67: Neither blocks nor log segments found. Creating new log.
20:03:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.499222  6431 log.cc:826] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67: Log is configured to *not* fsync() on all Append() calls
20:03:22.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.510989  6431 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67: No bootstrap required, opened a new log
20:03:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.536929  6431 raft_consensus.cc:359] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } }
20:03:22.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.537686  6431 raft_consensus.cc:385] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:22.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.538008  6431 raft_consensus.cc:740] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73a242ff54b84b15a1f7b1d5d9029e67, State: Initialized, Role: FOLLOWER
20:03:22.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.539175  6431 consensus_queue.cc:260] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } }
20:03:22.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.542614  6535 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } } }
20:03:22.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.543905  6535 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [sys.catalog]: This master's current role is: FOLLOWER
20:03:22.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.546306  6431 sys_catalog.cc:565] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [sys.catalog]: configured and running, proceeding with master startup.
20:03:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.546495  6406 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e0b84cba6b4f4afa8fd74c677579c439" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" is_pre_election: true
20:03:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.547456  6406 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e0b84cba6b4f4afa8fd74c677579c439 in term 0.
20:03:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.549053  6460 leader_election.cc:304] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73a242ff54b84b15a1f7b1d5d9029e67, e0b84cba6b4f4afa8fd74c677579c439; no voters: 
20:03:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.550653  6520 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:03:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.551182  6520 raft_consensus.cc:493] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:03:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.551622  6520 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 0 FOLLOWER]: Advancing to term 1
20:03:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.563917  6520 raft_consensus.cc:515] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } }
20:03:22.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.568791  6406 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e0b84cba6b4f4afa8fd74c677579c439" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73a242ff54b84b15a1f7b1d5d9029e67"
20:03:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.569535  6406 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [term 0 FOLLOWER]: Advancing to term 1
20:03:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.567540  6520 leader_election.cc:290] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [CANDIDATE]: Term 1 election: Requested vote from peers 3dcc02af18cc424e80c917ff88a021ad (127.6.32.254:43103), 73a242ff54b84b15a1f7b1d5d9029e67 (127.6.32.253:39891)
20:03:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.580483  6406 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b84cba6b4f4afa8fd74c677579c439 in term 1.
20:03:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.582156  6460 leader_election.cc:304] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 73a242ff54b84b15a1f7b1d5d9029e67, e0b84cba6b4f4afa8fd74c677579c439; no voters: 
20:03:22.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.583475  6520 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 1 FOLLOWER]: Leader election won for term 1
20:03:22.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.594239  6520 raft_consensus.cc:697] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [term 1 LEADER]: Becoming Leader. State: Replica: e0b84cba6b4f4afa8fd74c677579c439, State: Running, Role: LEADER
20:03:22.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.596397  6520 consensus_queue.cc:237] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } }
20:03:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.621333  6538 sys_catalog.cc:455] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e0b84cba6b4f4afa8fd74c677579c439. Latest consensus state: current_term: 1 leader_uuid: "e0b84cba6b4f4afa8fd74c677579c439" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } } }
20:03:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.622612  6538 sys_catalog.cc:458] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [sys.catalog]: This master's current role is: LEADER
20:03:22.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.635871  6545 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
20:03:22.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.643647  6545 catalog_manager.cc:1520] Initializing Kudu cluster ID...
20:03:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.672801  6406 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [term 1 FOLLOWER]: Refusing update from remote peer e0b84cba6b4f4afa8fd74c677579c439: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.675834  6538 consensus_queue.cc:1048] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:22.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:22.694888  6549 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:22.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:22.695827  6549 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:22.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.664554  6358 master_runner.cc:187] Error getting master registration for 127.6.32.254:43103: OK, instance_id { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" instance_seqno: 1783540998054328 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:22.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.717334  6358 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:03:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:22.739697  6385 tablet.cc:2406] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.746068  6535 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e0b84cba6b4f4afa8fd74c677579c439. Latest consensus state: current_term: 1 leader_uuid: "e0b84cba6b4f4afa8fd74c677579c439" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } } }
20:03:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.746973  6535 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [sys.catalog]: This master's current role is: FOLLOWER
20:03:22.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.753252  6550 mvcc.cc:204] Tried to move back new op lower bound from 7305383946910658560 to 7305383946649411584. Current Snapshot: MvccSnapshot[applied={T|T < 7305383946910658560}]
20:03:22.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.757071  6520 sys_catalog.cc:455] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e0b84cba6b4f4afa8fd74c677579c439" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } } }
20:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.758375  6520 sys_catalog.cc:458] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [sys.catalog]: This master's current role is: LEADER
20:03:22.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.761626  6535 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e0b84cba6b4f4afa8fd74c677579c439" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } } }
20:03:22.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.762331  6535 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67 [sys.catalog]: This master's current role is: FOLLOWER
20:03:22.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.766992  6356 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } attempt: 7
20:03:22.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.768110  6545 catalog_manager.cc:1383] Generated new cluster ID: 2517ddb150b840549f51ac02b130d086
20:03:22.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.768440  6545 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
20:03:22.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.775060  6552 mvcc.cc:204] Tried to move back new op lower bound from 7305383946910658560 to 7305383946649411584. Current Snapshot: MvccSnapshot[applied={T|T < 7305383946910658560 or (T in {7305383946910658560})}]
20:03:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.816154  6356 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad: Bootstrap starting.
20:03:22.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.848858  6356 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad: Neither blocks nor log segments found. Creating new log.
20:03:22.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.851898  6356 log.cc:826] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad: Log is configured to *not* fsync() on all Append() calls
20:03:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.879660  6545 catalog_manager.cc:1406] Generated new certificate authority record
20:03:22.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.884143  6356 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad: No bootstrap required, opened a new log
20:03:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.887080  6545 catalog_manager.cc:1540] Loading token signing keys...
20:03:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.926914  6356 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } }
20:03:22.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.928340  6356 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:22.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.928884  6356 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dcc02af18cc424e80c917ff88a021ad, State: Initialized, Role: FOLLOWER
20:03:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.930305  6356 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } }
20:03:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.938712  6558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } } }
20:03:22.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.940819  6558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [sys.catalog]: This master's current role is: FOLLOWER
20:03:22.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.950917  6545 catalog_manager.cc:6070] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439: Generated new TSK 0
20:03:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.952111  6545 catalog_manager.cc:1550] Initializing in-progress tserver states...
20:03:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.969619  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data
20:03:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/logs
20:03:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.193:0
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.193
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.193
20:03:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:22.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.985606  6356 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [sys.catalog]: configured and running, proceeding with master startup.
20:03:22.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.985795  6330 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e0b84cba6b4f4afa8fd74c677579c439" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dcc02af18cc424e80c917ff88a021ad"
20:03:22.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.986758  6330 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [term 0 FOLLOWER]: Advancing to term 1
20:03:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:22.991843  6331 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e0b84cba6b4f4afa8fd74c677579c439" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dcc02af18cc424e80c917ff88a021ad" is_pre_election: true
20:03:23.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.001504  6330 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b84cba6b4f4afa8fd74c677579c439 in term 1.
20:03:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.003809  6329 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [term 1 FOLLOWER]: Refusing update from remote peer e0b84cba6b4f4afa8fd74c677579c439: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:23.009261  6310 tablet.cc:2406] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.009012  6538 consensus_queue.cc:1048] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:03:23.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.113562  6558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [sys.catalog]: SysCatalogTable state changed. Reason: New leader e0b84cba6b4f4afa8fd74c677579c439. Latest consensus state: current_term: 1 leader_uuid: "e0b84cba6b4f4afa8fd74c677579c439" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } } }
20:03:23.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.113636  6566 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e0b84cba6b4f4afa8fd74c677579c439" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } } }
20:03:23.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.114384  6558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [sys.catalog]: This master's current role is: FOLLOWER
20:03:23.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.114466  6566 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad [sys.catalog]: This master's current role is: FOLLOWER
20:03:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.139778  6576 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:03:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:23.442489  6559 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:23.443148  6559 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:23.443884  6559 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:23.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:23.481215  6559 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:23.482282  6559 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.193
20:03:23.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.526768  6538 sys_catalog.cc:455] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e0b84cba6b4f4afa8fd74c677579c439" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dcc02af18cc424e80c917ff88a021ad" member_type: VOTER last_known_addr { host: "127.6.32.254" port: 43103 } } peers { permanent_uuid: "73a242ff54b84b15a1f7b1d5d9029e67" member_type: VOTER last_known_addr { host: "127.6.32.253" port: 39891 } } peers { permanent_uuid: "e0b84cba6b4f4afa8fd74c677579c439" member_type: VOTER last_known_addr { host: "127.6.32.252" port: 42851 } } }
20:03:23.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.528776  6538 sys_catalog.cc:458] T 00000000000000000000000000000000 P e0b84cba6b4f4afa8fd74c677579c439 [sys.catalog]: This master's current role is: LEADER
20:03:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.541327  6559 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.193:0
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.193
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.193
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/logs
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.543223  6559 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.546048  6559 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:23.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:23.565817  6584 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:23.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.701802  6549 catalog_manager.cc:1295] Loaded cluster ID: 2517ddb150b840549f51ac02b130d086
20:03:23.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.702572  6549 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67: loading cluster ID for follower catalog manager: success
20:03:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.710429  6549 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67: acquiring CA information for follower catalog manager: success
20:03:23.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:23.728200  6549 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 73a242ff54b84b15a1f7b1d5d9029e67: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:24.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:24.159387  6576 catalog_manager.cc:1295] Loaded cluster ID: 2517ddb150b840549f51ac02b130d086
20:03:24.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:24.159821  6576 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad: loading cluster ID for follower catalog manager: success
20:03:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:24.175144  6576 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad: acquiring CA information for follower catalog manager: success
20:03:24.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:24.186291  6576 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 3dcc02af18cc424e80c917ff88a021ad: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:24.967890  6583 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 6559
20:03:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:26.199590  6583 kernel_stack_watchdog.cc:198] Thread 6559 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
20:03:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:03:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:03:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:03:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:03:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:23.570497  6585 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:26.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:26.206308  6559 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.638s	user 0.859s	sys 1.286s
20:03:26.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:26.207775  6559 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.639s	user 0.859s	sys 1.286s
20:03:26.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:26.214226  6591 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.215562  6559 server_base.cc:1061] running on GCE node
20:03:26.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.216792  6559 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:26.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.220424  6559 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:26.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.222222  6559 hybrid_clock.cc:648] HybridClock initialized: now 1783541006222141 us; error 95 us; skew 500 ppm
20:03:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.223217  6559 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:26.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.238435  6559 webserver.cc:533] Webserver started at http://127.6.32.193:39967/ using document root <none> and password file <none>
20:03:26.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.239758  6559 fs_manager.cc:362] Metadata directory not provided
20:03:26.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.240111  6559 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:26.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.240679  6559 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.245167  6559 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/instance:
20:03:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a9f83cc3ec74be2b5c075c8011c00f4"
20:03:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:26 on dist-test-slave-xfg5"
20:03:26.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.246493  6559 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal/instance:
20:03:26.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a9f83cc3ec74be2b5c075c8011c00f4"
20:03:26.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:26 on dist-test-slave-xfg5"
20:03:26.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.257457  6559 fs_manager.cc:696] Time spent creating directory manager: real 0.010s	user 0.007s	sys 0.000s
20:03:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.264727  6596 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.266052  6559 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.006s	sys 0.000s
20:03:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.266445  6559 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a9f83cc3ec74be2b5c075c8011c00f4"
20:03:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:26 on dist-test-slave-xfg5"
20:03:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.266911  6559 fs_report.cc:389] FS layout report
20:03:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/data
20:03:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:26.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.321208  6559 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:26.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.322992  6559 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:26.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.323532  6559 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.326893  6559 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:26.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.332034  6559 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:26.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.332293  6559 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:03:26.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.332592  6559 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:26.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.332878  6559 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:03:26.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.538985  6559 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.193:39013
20:03:26.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.539191  6709 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.193:39013 every 8 connection(s)
20:03:26.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.542040  6559 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:03:26.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.547015  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 6559
20:03:26.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.547753  6275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal/instance
20:03:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.564021  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:03:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/logs
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.194:0
20:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.194
20:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.194
20:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:26.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.611213  6711 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:03:26.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.611868  6711 heartbeater.cc:461] Registering TS with master...
20:03:26.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.613459  6711 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:03:26.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.614907  6710 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:03:26.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.615393  6710 heartbeater.cc:461] Registering TS with master...
20:03:26.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.616427  6710 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:03:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.620465  6396 ts_manager.cc:194] Registered new tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:26.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.623435  6476 ts_manager.cc:194] Registered new tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:26.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.626116  6476 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.193:47327
20:03:26.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.646442  6717 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:03:26.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.646955  6717 heartbeater.cc:461] Registering TS with master...
20:03:26.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.647656  6717 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:03:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:26.650898  6320 ts_manager.cc:194] Registered new tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:26.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:26.971444  6716 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:26.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:26.972064  6716 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:26.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:26.972680  6716 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:27.011940  6716 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:27.012990  6716 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.194
20:03:27.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:27.053104  6716 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:27.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data
20:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.194:0
20:03:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.194
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.194
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/logs
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:27.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:27.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:27.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:27.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:27.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:27.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:27.054973  6716 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:27.057355  6716 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:27.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:27.075524  6726 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:27.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:27.635030  6711 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:03:28.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:27.092149  6727 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:28.449018  6716 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.362s	user 0.541s	sys 0.806s
20:03:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:28.449954  6716 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.363s	user 0.542s	sys 0.806s
20:03:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.452418  6716 server_base.cc:1061] running on GCE node
20:03:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:28.454579  6733 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:28.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.456243  6716 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.459378  6716 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.460891  6716 hybrid_clock.cc:648] HybridClock initialized: now 1783541008460816 us; error 57 us; skew 500 ppm
20:03:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.462158  6716 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:28.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.471899  6716 webserver.cc:533] Webserver started at http://127.6.32.194:33139/ using document root <none> and password file <none>
20:03:28.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.473256  6716 fs_manager.cc:362] Metadata directory not provided
20:03:28.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.473589  6716 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:28.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.474234  6716 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:28.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.481492  6716 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/instance:
20:03:28.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:03:28.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:28 on dist-test-slave-xfg5"
20:03:28.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.483258  6716 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal/instance:
20:03:28.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:03:28.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:28 on dist-test-slave-xfg5"
20:03:28.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.495012  6716 fs_manager.cc:696] Time spent creating directory manager: real 0.011s	user 0.010s	sys 0.001s
20:03:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.502276  6738 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.503638  6716 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.007s	sys 0.000s
20:03:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.504060  6716 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:03:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:03:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:28 on dist-test-slave-xfg5"
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.504489  6716 fs_report.cc:389] FS layout report
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/data
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.591682  6716 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:28.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.593339  6716 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:28.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.593923  6716 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:28.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.596823  6716 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:28.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.601620  6716 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:28.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.601904  6716 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:03:28.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.602205  6716 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:28.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.602485  6716 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:03:28.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.765504  6716 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.194:41701
20:03:28.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.765614  6850 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.194:41701 every 8 connection(s)
20:03:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.768313  6716 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:03:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.774026  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 6716
20:03:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.774655  6275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal/instance
20:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.805604  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data
20:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/logs
20:03:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:03:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.195:0
20:03:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.195
20:03:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.195
20:03:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:28.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:28.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:28.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.826186  6852 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:03:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.826875  6852 heartbeater.cc:461] Registering TS with master...
20:03:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.827782  6851 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:03:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.828152  6851 heartbeater.cc:461] Registering TS with master...
20:03:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.828660  6852 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:03:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.829622  6851 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:03:28.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.832275  6476 ts_manager.cc:194] Registered new tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:03:28.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.833603  6476 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.194:53429
20:03:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.833879  6396 ts_manager.cc:194] Registered new tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:03:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.845300  6853 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:03:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.845613  6853 heartbeater.cc:461] Registering TS with master...
20:03:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.846099  6853 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:03:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:28.848796  6320 ts_manager.cc:194] Registered new tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:03:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:29.185874  6859 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:29.186380  6859 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:29.187098  6859 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:29.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:29.227059  6859 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:29.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:29.229049  6859 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.195
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:29.270746  6859 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.195:0
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.195
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.195
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/logs
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:29.272089  6859 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:29.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:29.273804  6859 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:29.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:29.288537  6867 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:29.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:29.838120  6852 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:03:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:30.693360  6866 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 6859
20:03:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:31.012229  6866 kernel_stack_watchdog.cc:198] Thread 6859 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
20:03:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:03:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:03:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:03:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:03:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:29.289739  6868 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:31.012804  6859 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.724s	user 0.607s	sys 1.071s
20:03:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:31.014328  6859 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.726s	user 0.607s	sys 1.071s
20:03:31.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:31.015754  6870 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:31.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:31.020645  6869 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1728 milliseconds
20:03:31.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.020676  6859 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:31.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.022629  6859 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:31.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.024901  6859 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:31.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.026296  6859 hybrid_clock.cc:648] HybridClock initialized: now 1783541011026230 us; error 66 us; skew 500 ppm
20:03:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.027503  6859 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:31.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.034127  6859 webserver.cc:533] Webserver started at http://127.6.32.195:36303/ using document root <none> and password file <none>
20:03:31.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.035101  6859 fs_manager.cc:362] Metadata directory not provided
20:03:31.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.035301  6859 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:31.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.035667  6859 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:31.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.041007  6859 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/instance:
20:03:31.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e361d22f8464508ab46b5ee78fb2a3a"
20:03:31.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:31 on dist-test-slave-xfg5"
20:03:31.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.042805  6859 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal/instance:
20:03:31.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e361d22f8464508ab46b5ee78fb2a3a"
20:03:31.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:31 on dist-test-slave-xfg5"
20:03:31.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.053002  6859 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.004s	sys 0.004s
20:03:31.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.059438  6878 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.060738  6859 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
20:03:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.061154  6859 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e361d22f8464508ab46b5ee78fb2a3a"
20:03:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:31 on dist-test-slave-xfg5"
20:03:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.061540  6859 fs_report.cc:389] FS layout report
20:03:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/data
20:03:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:31.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.132596  6859 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.134289  6859 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:31.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.134866  6859 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:31.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.137591  6859 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:31.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.142853  6859 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:31.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.143108  6859 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:03:31.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.143415  6859 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:31.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.143631  6859 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:03:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.318318  6859 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.195:39759
20:03:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.318444  6990 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.195:39759 every 8 connection(s)
20:03:31.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.320961  6859 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:03:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.328774  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 6859
20:03:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.329393  6275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal/instance
20:03:31.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.362622  6992 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:03:31.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.363741  6992 heartbeater.cc:461] Registering TS with master...
20:03:31.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.365931  6992 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:03:31.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.366381  6991 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:03:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.366765  6991 heartbeater.cc:461] Registering TS with master...
20:03:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.367671  6991 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:03:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.368569  6993 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:03:31.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.368916  6993 heartbeater.cc:461] Registering TS with master...
20:03:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.370031  6993 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:03:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.370069  6476 ts_manager.cc:194] Registered new tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:31.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.371789  6476 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.195:49007
20:03:31.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.372274  6396 ts_manager.cc:194] Registered new tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.372574  6320 ts_manager.cc:194] Registered new tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:31.391726  6275 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:03:31.407 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
20:03:31.408 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

20:03:31.418 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.6.32.254"
      port: 43103
    }
    bound_http_address {
      host: "127.6.32.254"
      port: 36611
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.6.32.253"
      port: 39891
    }
    bound_http_address {
      host: "127.6.32.253"
      port: 45589
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.6.32.252"
      port: 42851
    }
    bound_http_address {
      host: "127.6.32.252"
      port: 33911
    }
  }
}

20:03:31.421 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

20:03:31.424 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.6.32.193"
      port: 39013
    }
    bound_http_address {
      host: "127.6.32.193"
      port: 39967
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.6.32.194"
      port: 41701
    }
    bound_http_address {
      host: "127.6.32.194"
      port: 33139
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.6.32.195"
      port: 39759
    }
    bound_http_address {
      host: "127.6.32.195"
      port: 36303
    }
  }
}

20:03:31.425 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:03:31.604 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
20:03:31.663 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
20:03:31.844 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.32.252:42851(127.6.32.252:42851)] connecting to peer
20:03:31.909 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.32.253:39891(127.6.32.253:39891)] connecting to peer
20:03:31.911 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.32.254:43103(127.6.32.254:43103)] connecting to peer
20:03:31.929 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.6.32.254:43103(127.6.32.254:43103)] Successfully connected to peer
20:03:31.929 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.6.32.253:39891(127.6.32.253:39891)] Successfully connected to peer
20:03:31.933 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.6.32.252:42851(127.6.32.252:42851)] Successfully connected to peer
20:03:32.045 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.6.32.254
20:03:32.045 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.6.32.252
20:03:32.045 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.6.32.253
20:03:32.293 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xe3c04d6d, L:/127.0.0.1:40862 - R:/127.6.32.253:39891] using SASL/PLAIN
20:03:32.293 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xeac1cad2, L:/127.0.0.1:56244 - R:/127.6.32.252:42851] using SASL/PLAIN
20:03:32.295 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x67c11272, L:/127.0.0.1:51984 - R:/127.6.32.254:43103] using SASL/PLAIN
20:03:32.334 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:03:32.336 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=533), Traces: [0ms] refreshing cache from master, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.32.252:42851, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.32.253:39891, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.32.254:43103, [501ms] Sub RPC ConnectToMaster: received response from server master-127.6.32.253:39891: OK, [501ms] Sub RPC ConnectToMaster: received response from server master-127.6.32.254:43103: OK, [502ms] Sub RPC ConnectToMaster: received response from server master-127.6.32.252:42851: OK, deferred=null) after lookup
20:03:32.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.375494  6992 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:03:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.393450  6476 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56244:
20:03:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789"
20:03:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:03:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
20:03:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:03:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
20:03:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
20:03:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
20:03:32.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:32.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:03:32.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:32.398391  6476 catalog_manager.cc:7077] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:03:32.477 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 54141570ab294d839ebb133abb74fd45
20:03:32.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.501315  6786 tablet_service.cc:1511] Processing CreateTablet for tablet a41f0e13a5c84a95831681171defffc3 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789 [id=54141570ab294d839ebb133abb74fd45]), partition=RANGE (key) PARTITION UNBOUNDED
20:03:32.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.503624  6786 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a41f0e13a5c84a95831681171defffc3. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:32.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.518613  6926 tablet_service.cc:1511] Processing CreateTablet for tablet a41f0e13a5c84a95831681171defffc3 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789 [id=54141570ab294d839ebb133abb74fd45]), partition=RANGE (key) PARTITION UNBOUNDED
20:03:32.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.520537  6926 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a41f0e13a5c84a95831681171defffc3. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.516872  6644 tablet_service.cc:1511] Processing CreateTablet for tablet a41f0e13a5c84a95831681171defffc3 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789 [id=54141570ab294d839ebb133abb74fd45]), partition=RANGE (key) PARTITION UNBOUNDED
20:03:32.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.523869  6644 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a41f0e13a5c84a95831681171defffc3. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.554384  7025 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Bootstrap starting.
20:03:32.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.563071  7025 tablet_bootstrap.cc:654] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Neither blocks nor log segments found. Creating new log.
20:03:32.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.565672  7026 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap starting.
20:03:32.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.567678  7025 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Log is configured to *not* fsync() on all Append() calls
20:03:32.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.576639  7026 tablet_bootstrap.cc:654] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Neither blocks nor log segments found. Creating new log.
20:03:32.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.579058  7026 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Log is configured to *not* fsync() on all Append() calls
20:03:32.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.581897  7025 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: No bootstrap required, opened a new log
20:03:32.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.583081  7025 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Time spent bootstrapping tablet: real 0.031s	user 0.013s	sys 0.010s
20:03:32.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.600000  7028 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Bootstrap starting.
20:03:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.602392  7026 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: No bootstrap required, opened a new log
20:03:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.603420  7026 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Time spent bootstrapping tablet: real 0.038s	user 0.019s	sys 0.012s
20:03:32.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.613785  7028 tablet_bootstrap.cc:654] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Neither blocks nor log segments found. Creating new log.
20:03:32.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.616794  7028 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Log is configured to *not* fsync() on all Append() calls
20:03:32.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.618655  7025 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:32.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.619987  7025 raft_consensus.cc:385] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:32.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.620505  7025 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ba53d199a5c47b6acb057a49d2d0699, State: Initialized, Role: FOLLOWER
20:03:32.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.621749  7025 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.627004  7025 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Time spent starting tablet: real 0.043s	user 0.029s	sys 0.007s
20:03:32.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.631613  7028 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: No bootstrap required, opened a new log
20:03:32.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.632604  7028 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Time spent bootstrapping tablet: real 0.034s	user 0.010s	sys 0.015s
20:03:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.640596  7026 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.642314  7026 raft_consensus.cc:385] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:32.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.642642  7026 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e361d22f8464508ab46b5ee78fb2a3a, State: Initialized, Role: FOLLOWER
20:03:32.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.643831  7026 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.654650  7026 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Time spent starting tablet: real 0.051s	user 0.030s	sys 0.013s
20:03:32.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.658102  7032 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:32.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.659058  7032 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:32.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.667908  7028 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:32.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.668859  7028 raft_consensus.cc:385] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:32.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.669155  7028 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a9f83cc3ec74be2b5c075c8011c00f4, State: Initialized, Role: FOLLOWER
20:03:32.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.670097  7028 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:32.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.672781  7032 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.676869  7028 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Time spent starting tablet: real 0.044s	user 0.023s	sys 0.015s
20:03:32.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.703701  6806 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699" is_pre_election: true
20:03:32.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.704658  6806 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e361d22f8464508ab46b5ee78fb2a3a in term 0.
20:03:32.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.705652  6664 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" is_pre_election: true
20:03:32.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.706074  6881 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8ba53d199a5c47b6acb057a49d2d0699, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 
20:03:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.706565  6664 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e361d22f8464508ab46b5ee78fb2a3a in term 0.
20:03:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.707178  7032 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 0 FOLLOWER]: Leader pre-election won for term 1
20:03:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.707551  7032 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:03:32.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.707891  7032 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 0 FOLLOWER]: Advancing to term 1
20:03:32.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.713681  7032 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:32.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.715250  7032 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 1 election: Requested vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:32.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.716117  6806 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:03:32.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.716246  6664 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4"
20:03:32.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.716518  6806 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 0 FOLLOWER]: Advancing to term 1
20:03:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.716789  6664 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 0 FOLLOWER]: Advancing to term 1
20:03:32.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.722081  6806 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e361d22f8464508ab46b5ee78fb2a3a in term 1.
20:03:32.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.722251  6664 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e361d22f8464508ab46b5ee78fb2a3a in term 1.
20:03:32.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.723237  6881 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8ba53d199a5c47b6acb057a49d2d0699, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 
20:03:32.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.724006  7032 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 1 FOLLOWER]: Leader election won for term 1
20:03:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.725977  7032 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 1 LEADER]: Becoming Leader. State: Replica: 9e361d22f8464508ab46b5ee78fb2a3a, State: Running, Role: LEADER
20:03:32.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.726949  7032 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:32.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.739081  6474 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a reported cstate change: term changed from 0 to 1, leader changed from <none> to 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195). New cstate: current_term: 1 leader_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: HEALTHY } } }
20:03:32.815 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 54141570ab294d839ebb133abb74fd45
20:03:32.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:32.825876  6858 tablet.cc:2406] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:32.835 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:32.844 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299992, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:32.845 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [10ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=null) after lookup
20:03:32.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:32.852284  6998 tablet.cc:2406] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:32.859 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:32.861 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] Successfully connected to peer
20:03:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:32.893700  6718 tablet.cc:2406] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:32.938 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0xd0f00cf0, L:/127.0.0.1:38258 - R:/127.6.32.195:39759] using TOKEN/null
20:03:32.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.956032  6664 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 1 FOLLOWER]: Refusing update from remote peer 9e361d22f8464508ab46b5ee78fb2a3a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:32.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.956409  6806 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 1 FOLLOWER]: Refusing update from remote peer 9e361d22f8464508ab46b5ee78fb2a3a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:32.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.958140  7036 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:03:32.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:32.959043  7032 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:33.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:33.006525  7041 mvcc.cc:204] Tried to move back new op lower bound from 7305383989045854208 to 7305383988137533440. Current Snapshot: MvccSnapshot[applied={T|T < 7305383989045854208}]
20:03:33.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:33.024205  7042 mvcc.cc:204] Tried to move back new op lower bound from 7305383989045854208 to 7305383988137533440. Current Snapshot: MvccSnapshot[applied={T|T < 7305383989045854208}]
20:03:33.218 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:33.220 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.32.195:39759
20:03:33.221 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:33.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:33.222451  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 6859
20:03:33.245 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:33.247 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] handling channelInactive
20:03:33.248 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state READY due to: connection closed
20:03:33.248 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: connection closed
20:03:33.269 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:33.270 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:33.270 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [23ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@2031712225(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@754d6fa1 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fc651af -> wakeup thread Time-limited test)) after lookup
20:03:33.271 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:33.273 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.273 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.273 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.290 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:33.290 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:33.291 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [23ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [24ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [27ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [28ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [43ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@2031712225(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@754d6fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1964322125 after retry RPC@1864371252), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fc651af -> wakeup thread Time-limited test -> (continuation of Deferred@1964322125 after retry RPC@1864371252))) after lookup
20:03:33.292 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:33.295 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.295 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.295 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.310 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:33.310 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:33.311 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=64), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [23ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [24ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [27ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [28ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [43ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [46ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [49ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [49ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [63ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@2031712225(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@754d6fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@817345663 after retry RPC@981336283), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fc651af -> wakeup thread Time-limited test -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@817345663 after retry RPC@981336283))) after lookup
20:03:33.312 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:33.314 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.314 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.314 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.450 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:33.450 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:33.451 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=205), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [23ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [24ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [27ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [28ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [43ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [46ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [49ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [49ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [63ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [65ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [68ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [69ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [198ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [204ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@2031712225(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@754d6fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@817345663 after retry RPC@981336283) -> (continuation of Deferred@945832304 after retry RPC@1754617109), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fc651af -> wakeup thread Time-limited test -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@817345663 after retry RPC@981336283) -> (continuation of Deferred@945832304 after retry RPC@1754617109))) after lookup
20:03:33.452 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:33.455 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.455 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.455 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.970 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:33.970 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:33.971 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=724), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [23ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [24ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [27ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [28ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [43ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [46ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [49ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [49ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [63ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [65ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [68ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [69ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [198ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [204ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [206ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [209ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [210ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [718ms] refreshing cache from master, [719ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [723ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@2031712225(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@754d6fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@817345663 after retry RPC@981336283) -> (continuation of Deferred@945832304 after retry RPC@1754617109) -> (continuation of Deferred@1465584654 after retry RPC@1180131173), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fc651af -> wakeup thread Time-limited test -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@817345663 after retry RPC@981336283) -> (continuation of Deferred@945832304 after retry RPC@1754617109) -> (continuation of Deferred@1465584654 after retry RPC@1180131173))) after lookup
20:03:33.972 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:33.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:33.973 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.645176  7057 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 9e361d22f8464508ab46b5ee78fb2a3a)
20:03:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.646138  7057 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.648877  7057 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.654201  7061 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 9e361d22f8464508ab46b5ee78fb2a3a)
20:03:34.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:34.654562  6600 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:03:34.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.655490  7061 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:34.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:34.661872  6600 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:03:34.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.663414  7061 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:34.668984  6742 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:03:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.668835  6806 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699" is_pre_election: true
20:03:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.669679  6806 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a9f83cc3ec74be2b5c075c8011c00f4 in term 1.
20:03:34.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.674450  6599 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 8ba53d199a5c47b6acb057a49d2d0699; no voters: 9e361d22f8464508ab46b5ee78fb2a3a
20:03:34.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.675729  7057 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:03:34.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.676118  7057 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 9e361d22f8464508ab46b5ee78fb2a3a)
20:03:34.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.676395  7057 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 1 FOLLOWER]: Advancing to term 2
20:03:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:34.678000  6742 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:03:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.678159  6664 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "8ba53d199a5c47b6acb057a49d2d0699" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" is_pre_election: true
20:03:34.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.681879  7057 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:34.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.682972  6664 raft_consensus.cc:2393] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 8ba53d199a5c47b6acb057a49d2d0699 in current term 2: Already voted for candidate 1a9f83cc3ec74be2b5c075c8011c00f4 in this term.
20:03:34.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.683795  7057 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 2 election: Requested vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:34.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.684948  6806 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:03:34.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.685719  6806 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 1 FOLLOWER]: Advancing to term 2
20:03:34.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:34.690264  6600 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:03:34.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.690191  6742 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8ba53d199a5c47b6acb057a49d2d0699; no voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 9e361d22f8464508ab46b5ee78fb2a3a
20:03:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.692669  6806 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a9f83cc3ec74be2b5c075c8011c00f4 in term 2.
20:03:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.693109  7061 raft_consensus.cc:2749] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:03:34.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.693699  6599 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 8ba53d199a5c47b6acb057a49d2d0699; no voters: 9e361d22f8464508ab46b5ee78fb2a3a
20:03:34.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.694394  7057 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 2 FOLLOWER]: Leader election won for term 2
20:03:34.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.695992  7057 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 2 LEADER]: Becoming Leader. State: Replica: 1a9f83cc3ec74be2b5c075c8011c00f4, State: Running, Role: LEADER
20:03:34.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.696935  7057 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:34.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:34.706375  6475 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 reported cstate change: term changed from 1 to 2, leader changed from 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195) to 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193). New cstate: current_term: 2 leader_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: UNKNOWN } } }
20:03:35.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:35.190032  6600 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 -> Peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Couldn't send request to peer 9e361d22f8464508ab46b5ee78fb2a3a. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:35.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:35.305181  6806 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 2 FOLLOWER]: Refusing update from remote peer 1a9f83cc3ec74be2b5c075c8011c00f4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
20:03:35.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:35.306494  7057 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
20:03:35.989 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:35.989 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:35.990 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2744), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [23ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [24ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [27ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [28ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [43ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [46ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [49ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [49ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [63ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [65ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [68ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [69ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [198ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [204ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [206ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [209ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [210ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [718ms] refreshing cache from master, [719ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [723ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [725ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [727ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [727ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2738ms] refreshing cache from master, [2738ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [2742ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@2031712225(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@754d6fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@817345663 after retry RPC@981336283) -> (continuation of Deferred@945832304 after retry RPC@1754617109) -> (continuation of Deferred@1465584654 after retry RPC@1180131173) -> (continuation of Deferred@892069601 after retry RPC@424871685), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fc651af -> wakeup thread Time-limited test -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@817345663 after retry RPC@981336283) -> (continuation of Deferred@945832304 after retry RPC@1754617109) -> (continuation of Deferred@1465584654 after retry RPC@1180131173) -> (continuation of Deferred@892069601 after retry RPC@424871685))) after lookup
20:03:35.992 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:35.993 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] Successfully connected to peer
20:03:36.014 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x6031b207, L:/127.0.0.1:52012 - R:/127.6.32.193:39013] using TOKEN/null
20:03:36.073 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.32.195:39759
20:03:36.075 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:36.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:36.077486  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/logs
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.195:39759
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.195
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36303
20:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.195
20:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:36.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:36.408679  7093 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:36.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:36.409179  7093 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:36.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:36.409644  7093 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:36.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:36.444342  7093 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:36.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:36.445359  7093 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.195
20:03:36.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:36.483302  7093 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.195:39759
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.195
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36303
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.195
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/logs
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:36.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:36.485070  7093 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:36.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:36.487413  7093 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:36.501346  7102 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:37.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:37.773195  6600 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 -> Peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Couldn't send request to peer 9e361d22f8464508ab46b5ee78fb2a3a. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:36.501350  7103 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:37.906818  7101 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 7093
20:03:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:38.246402  7101 kernel_stack_watchdog.cc:198] Thread 7093 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:03:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:03:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:03:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:03:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:03:38.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:38.251547  7093 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.751s	user 0.519s	sys 1.126s
20:03:38.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:38.251961  7093 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.751s	user 0.519s	sys 1.126s
20:03:38.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:38.252990  7104 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1750 milliseconds
20:03:38.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:38.253520  7105 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.254025  7093 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.255548  7093 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.257790  7093 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.259135  7093 hybrid_clock.cc:648] HybridClock initialized: now 1783541018259074 us; error 67 us; skew 500 ppm
20:03:38.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.259895  7093 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.265761  7093 webserver.cc:533] Webserver started at http://127.6.32.195:36303/ using document root <none> and password file <none>
20:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.266798  7093 fs_manager.cc:362] Metadata directory not provided
20:03:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.267004  7093 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.274786  7093 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.001s
20:03:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.279386  7118 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.280534  7093 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
20:03:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.280826  7093 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e361d22f8464508ab46b5ee78fb2a3a"
20:03:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:31 on dist-test-slave-xfg5"
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.282775  7093 fs_report.cc:389] FS layout report
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/data
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.336944  7093 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.338514  7093 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:38.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.339154  7093 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.342767  7093 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:38.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.350530  7127 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:38.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.362367  7093 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:38.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.362655  7093 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.014s	user 0.000s	sys 0.003s
20:03:38.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.362908  7093 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.368094  7093 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.368322  7093 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.002s
20:03:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.368924  7127 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap starting.
20:03:38.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.430900  7127 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Log is configured to *not* fsync() on all Append() calls
20:03:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.463729  7127 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.464666  7127 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap complete.
20:03:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.466233  7127 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Time spent bootstrapping tablet: real 0.098s	user 0.060s	sys 0.030s
20:03:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.484346  7127 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.487437  7127 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e361d22f8464508ab46b5ee78fb2a3a, State: Initialized, Role: FOLLOWER
20:03:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.488687  7127 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.493052  7127 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Time spent starting tablet: real 0.027s	user 0.021s	sys 0.008s
20:03:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:38.508427  6706 debug-util.cc:398] Leaking SignalData structure 0x7b08000cc520 after lost signal to thread 6577
20:03:38.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:38.509850  6706 debug-util.cc:398] Leaking SignalData structure 0x7b08000c9900 after lost signal to thread 6709
20:03:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.563390  7093 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.195:39759
20:03:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.563680  7238 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.195:39759 every 8 connection(s)
20:03:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.567381  7093 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:03:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.575850  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 7093
20:03:38.576 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:38.611 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:38.613 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.32.193:39013
20:03:38.613 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:38.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.614042  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 6559
20:03:38.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.627537  7239 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:03:38.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.628090  7239 heartbeater.cc:461] Registering TS with master...
20:03:38.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.629124  7240 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:03:38.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.629487  7240 heartbeater.cc:461] Registering TS with master...
20:03:38.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.629570  7239 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:03:38.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.630450  7240 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:03:38.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.630681  7241 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:03:38.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.631096  7241 heartbeater.cc:461] Registering TS with master...
20:03:38.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.632040  7241 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:03:38.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.634557  6474 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:38.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.635280  6395 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:38.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.636240  6321 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:38.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:38.638056  6474 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.195:52115
20:03:38.653 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] handling channelInactive
20:03:38.653 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:38.653 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state READY due to: connection closed
20:03:38.654 [DEBUG - Time-limited test] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:38.655 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.655 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.655 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.669 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:38.669 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:38.670 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [16ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@162116517(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@8c3f6ba -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5002c23a -> wakeup thread Time-limited test)) after lookup
20:03:38.671 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:38.672 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.672 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.672 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.689 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:38.690 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:38.690 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [16ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [17ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [19ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [19ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [36ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@162116517(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@8c3f6ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5002c23a -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245))) after lookup
20:03:38.691 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:38.692 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.692 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.692 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.729 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:38.729 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:38.730 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=76), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [16ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [17ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [19ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [19ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [36ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [37ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [39ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [39ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [76ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@162116517(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@8c3f6ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245) -> (continuation of Deferred@1506597878 after retry RPC@1942470659), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5002c23a -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245) -> (continuation of Deferred@1506597878 after retry RPC@1942470659))) after lookup
20:03:38.731 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:38.733 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.733 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.734 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.789 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:38.790 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:38.790 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=136), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [16ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [17ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [19ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [19ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [36ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [37ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [39ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [39ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [76ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [77ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [81ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [81ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [136ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@162116517(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@8c3f6ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245) -> (continuation of Deferred@1506597878 after retry RPC@1942470659) -> (continuation of Deferred@620361110 after retry RPC@1769205458), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5002c23a -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245) -> (continuation of Deferred@1506597878 after retry RPC@1942470659) -> (continuation of Deferred@620361110 after retry RPC@1769205458))) after lookup
20:03:38.791 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:38.792 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.792 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.792 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.988 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:38.989 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:38.989 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=336), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [16ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [17ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [19ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [19ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [36ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [37ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [39ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [39ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [76ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [77ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [81ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [81ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [136ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [137ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [139ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [139ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [331ms] refreshing cache from master, [331ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [335ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@162116517(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@8c3f6ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245) -> (continuation of Deferred@1506597878 after retry RPC@1942470659) -> (continuation of Deferred@620361110 after retry RPC@1769205458) -> (continuation of Deferred@703431272 after retry RPC@1595866361), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5002c23a -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245) -> (continuation of Deferred@1506597878 after retry RPC@1942470659) -> (continuation of Deferred@620361110 after retry RPC@1769205458) -> (continuation of Deferred@703431272 after retry RPC@1595866361))) after lookup
20:03:38.990 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:38.992 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.992 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:38.992 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:39.408 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:39.409 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:39.409 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=755), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [16ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [17ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [19ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [19ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [36ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [37ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [39ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [39ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [76ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [77ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [81ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [81ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [136ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [137ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [139ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [139ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [331ms] refreshing cache from master, [331ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [335ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [337ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [339ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [339ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [751ms] refreshing cache from master, [751ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [755ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@162116517(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@8c3f6ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245) -> (continuation of Deferred@1506597878 after retry RPC@1942470659) -> (continuation of Deferred@620361110 after retry RPC@1769205458) -> (continuation of Deferred@703431272 after retry RPC@1595866361) -> (continuation of Deferred@2110153621 after retry RPC@108097948), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5002c23a -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245) -> (continuation of Deferred@1506597878 after retry RPC@1942470659) -> (continuation of Deferred@620361110 after retry RPC@1769205458) -> (continuation of Deferred@703431272 after retry RPC@1595866361) -> (continuation of Deferred@2110153621 after retry RPC@108097948))) after lookup
20:03:39.410 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:39.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:39.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:39.411 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:39.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.619237  7250 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.619688  7250 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:39.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.622260  7250 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:39.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:39.626760  7123 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111)
20:03:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:39.631481  7123 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111)
20:03:39.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.635814  6806 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699" is_pre_election: true
20:03:39.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.637511  7121 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 8ba53d199a5c47b6acb057a49d2d0699
20:03:39.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.638373  7250 raft_consensus.cc:2749] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:03:39.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.641224  7240 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:03:39.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.936885  7255 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 1a9f83cc3ec74be2b5c075c8011c00f4)
20:03:39.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.937433  7255 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:39.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.939389  7255 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:39.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:39.944279  6742 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111)
20:03:39.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:39.948922  6742 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111)
20:03:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.954310  7193 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "8ba53d199a5c47b6acb057a49d2d0699" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" is_pre_election: true
20:03:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.955044  7193 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8ba53d199a5c47b6acb057a49d2d0699 in term 1.
20:03:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.956328  6742 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8ba53d199a5c47b6acb057a49d2d0699, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 1a9f83cc3ec74be2b5c075c8011c00f4
20:03:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.956961  7255 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 2 FOLLOWER]: Leader pre-election won for term 3
20:03:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.957288  7255 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 1a9f83cc3ec74be2b5c075c8011c00f4)
20:03:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.957515  7255 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 2 FOLLOWER]: Advancing to term 3
20:03:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.962787  7255 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:39.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.964564  7255 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 3 election: Requested vote from peers 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:39.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.966249  7193 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "8ba53d199a5c47b6acb057a49d2d0699" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "9e361d22f8464508ab46b5ee78fb2a3a"
20:03:39.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.966979  7193 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 1 FOLLOWER]: Advancing to term 3
20:03:39.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:39.968709  6742 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111)
20:03:39.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.974622  7193 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8ba53d199a5c47b6acb057a49d2d0699 in term 3.
20:03:39.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.975818  6742 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8ba53d199a5c47b6acb057a49d2d0699, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 1a9f83cc3ec74be2b5c075c8011c00f4
20:03:39.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.976579  7255 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 3 FOLLOWER]: Leader election won for term 3
20:03:39.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.978631  7255 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 3 LEADER]: Becoming Leader. State: Replica: 8ba53d199a5c47b6acb057a49d2d0699, State: Running, Role: LEADER
20:03:39.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.980508  7255 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:39.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:39.991415  6474 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 reported cstate change: term changed from 2 to 3, leader changed from 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193) to 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194). New cstate: current_term: 3 leader_uuid: "8ba53d199a5c47b6acb057a49d2d0699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: UNKNOWN } } }
20:03:40.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:40.365118  6742 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 -> Peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Couldn't send request to peer 1a9f83cc3ec74be2b5c075c8011c00f4. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:40.446059  7193 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 3 FOLLOWER]: Refusing update from remote peer 8ba53d199a5c47b6acb057a49d2d0699: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
20:03:40.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:40.447697  7255 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
20:03:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:43.241110  6742 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 -> Peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Couldn't send request to peer 1a9f83cc3ec74be2b5c075c8011c00f4. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:43.429 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:43.430 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:43.430 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=4777), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [16ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [17ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [19ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [19ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [36ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [37ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [39ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [39ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [76ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [77ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [81ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [81ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [136ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [137ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [139ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [139ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [331ms] refreshing cache from master, [331ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [335ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [337ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [339ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [339ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [751ms] refreshing cache from master, [751ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [755ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [757ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [758ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [758ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [4771ms] refreshing cache from master, [4771ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [4776ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@162116517(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@8c3f6ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245) -> (continuation of Deferred@1506597878 after retry RPC@1942470659) -> (continuation of Deferred@620361110 after retry RPC@1769205458) -> (continuation of Deferred@703431272 after retry RPC@1595866361) -> (continuation of Deferred@2110153621 after retry RPC@108097948) -> (continuation of Deferred@228422591 after retry RPC@1827370618), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5002c23a -> wakeup thread Time-limited test -> (continuation of Deferred@448519042 after retry RPC@1559157245) -> (continuation of Deferred@1506597878 after retry RPC@1942470659) -> (continuation of Deferred@620361110 after retry RPC@1769205458) -> (continuation of Deferred@703431272 after retry RPC@1595866361) -> (continuation of Deferred@2110153621 after retry RPC@108097948) -> (continuation of Deferred@228422591 after retry RPC@1827370618))) after lookup
20:03:43.431 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:43.433 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] Successfully connected to peer
20:03:43.460 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x4f6a86ac, L:/127.0.0.1:58908 - R:/127.6.32.194:41701] using TOKEN/null
20:03:43.519 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.32.193:39013
20:03:43.520 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:43.521914  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/logs
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:03:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.193:39013
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.193
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39967
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.193
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:43.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:43.843840  7300 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:43.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:43.844334  7300 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:43.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:43.844802  7300 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:43.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:43.881104  7300 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:43.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:43.881958  7300 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.193
20:03:43.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:43.920267  7300 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:43.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:43.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.193:39013
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.193
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39967
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.193
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/logs
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:43.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:43.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:43.921664  7300 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:43.923240  7300 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:43.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:43.936038  7306 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:45.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:45.210968  6742 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111) [suppressed 10 similar messages]
20:03:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:45.617170  6742 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 -> Peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Couldn't send request to peer 1a9f83cc3ec74be2b5c075c8011c00f4. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
20:03:45.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:45.341041  7305 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 7300
20:03:45.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:45.631518  7300 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.694s	user 0.544s	sys 1.138s
20:03:45.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:45.631942  7300 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.695s	user 0.544s	sys 1.138s
20:03:45.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:43.937600  7307 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:45.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:45.632932  7308 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1694 milliseconds
20:03:45.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:45.633919  7309 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:45.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.633846  7300 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:45.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.637365  7300 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:45.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.639609  7300 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:45.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.640993  7300 hybrid_clock.cc:648] HybridClock initialized: now 1783541025640966 us; error 62 us; skew 500 ppm
20:03:45.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.641813  7300 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:45.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.648684  7300 webserver.cc:533] Webserver started at http://127.6.32.193:39967/ using document root <none> and password file <none>
20:03:45.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.649777  7300 fs_manager.cc:362] Metadata directory not provided
20:03:45.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.650065  7300 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:45.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.658298  7300 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.000s	sys 0.006s
20:03:45.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.663535  7323 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:45.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.664646  7300 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
20:03:45.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.665042  7300 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a9f83cc3ec74be2b5c075c8011c00f4"
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:26 on dist-test-slave-xfg5"
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.666956  7300 fs_report.cc:389] FS layout report
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/data
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:45.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.732630  7300 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:45.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.734637  7300 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:45.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.735203  7300 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.737871  7300 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:45.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.744253  7330 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:45.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.754326  7300 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:45.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.754635  7300 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.012s	user 0.002s	sys 0.000s
20:03:45.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.754940  7300 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.760191  7300 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.760407  7300 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
20:03:45.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.760887  7330 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Bootstrap starting.
20:03:45.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.821738  7330 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Log is configured to *not* fsync() on all Append() calls
20:03:45.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.863950  7330 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.865128  7330 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Bootstrap complete.
20:03:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.866860  7330 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Time spent bootstrapping tablet: real 0.106s	user 0.090s	sys 0.015s
20:03:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.889815  7330 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.892395  7330 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a9f83cc3ec74be2b5c075c8011c00f4, State: Initialized, Role: FOLLOWER
20:03:45.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.893743  7330 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.898034  7330 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Time spent starting tablet: real 0.031s	user 0.022s	sys 0.008s
20:03:45.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.940125  7300 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.193:39013
20:03:45.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.940407  7438 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.193:39013 every 8 connection(s)
20:03:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.942795  7300 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:03:45.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.944283  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 7300
20:03:45.945 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:45.972 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:45.973 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.32.194:41701
20:03:45.973 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:45.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.974359  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 6716
20:03:45.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.996995  7439 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:03:45.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.997574  7439 heartbeater.cc:461] Registering TS with master...
20:03:45.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:45.998826  7439 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:03:46.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:46.001084  7440 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:03:46.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:46.001472  7440 heartbeater.cc:461] Registering TS with master...
20:03:46.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:46.002486  7440 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:03:46.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:46.003862  6395 ts_manager.cc:194] Re-registered known tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:46.007124  7441 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:03:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:46.007521  7441 heartbeater.cc:461] Registering TS with master...
20:03:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:46.007983  6475 ts_manager.cc:194] Re-registered known tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:46.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:46.008447  7441 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:03:46.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:46.011708  6475 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.193:43999
20:03:46.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:46.012318  6320 ts_manager.cc:194] Re-registered known tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:46.026 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] handling channelInactive
20:03:46.026 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state READY due to: connection closed
20:03:46.027 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:46.028 [DEBUG - Time-limited test] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:46.030 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.030 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.030 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.049 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:46.049 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:46.050 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [21ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@83709950(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@3392b1bd -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3bb02910 -> wakeup thread Time-limited test)) after lookup
20:03:46.051 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:46.052 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.052 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.052 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.069 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:46.069 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:46.070 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [21ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [22ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [24ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [24ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@83709950(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@3392b1bd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3bb02910 -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622))) after lookup
20:03:46.072 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:46.073 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.074 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.074 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.110 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:46.110 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:46.111 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=83), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [21ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [22ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [24ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [24ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [42ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [46ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [46ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [81ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@83709950(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@3392b1bd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3bb02910 -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985))) after lookup
20:03:46.112 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:46.113 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.113 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.113 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.230 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:46.230 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:46.230 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=202), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [21ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [22ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [24ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [24ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [42ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [46ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [46ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [81ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [84ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [85ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [85ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [196ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [201ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@83709950(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@3392b1bd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985) -> (continuation of Deferred@1288604902 after retry RPC@257302969), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3bb02910 -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985) -> (continuation of Deferred@1288604902 after retry RPC@257302969))) after lookup
20:03:46.231 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:46.232 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.232 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.232 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.249 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:46.249 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:46.249 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=221), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [21ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [22ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [24ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [24ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [42ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [46ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [46ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [81ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [84ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [85ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [85ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [196ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [201ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [203ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [204ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [204ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [216ms] refreshing cache from master, [216ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [220ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@83709950(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@3392b1bd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985) -> (continuation of Deferred@1288604902 after retry RPC@257302969) -> (continuation of Deferred@391011632 after retry RPC@690744713), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3bb02910 -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985) -> (continuation of Deferred@1288604902 after retry RPC@257302969) -> (continuation of Deferred@391011632 after retry RPC@690744713))) after lookup
20:03:46.250 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:46.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.251 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.251 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.749 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:46.749 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:46.750 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=721), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [21ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [22ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [24ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [24ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [42ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [46ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [46ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [81ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [84ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [85ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [85ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [196ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [201ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [203ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [204ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [204ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [216ms] refreshing cache from master, [216ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [220ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [222ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [223ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [223ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [716ms] refreshing cache from master, [717ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [721ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@83709950(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@3392b1bd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985) -> (continuation of Deferred@1288604902 after retry RPC@257302969) -> (continuation of Deferred@391011632 after retry RPC@690744713) -> (continuation of Deferred@1578138435 after retry RPC@673284781), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3bb02910 -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985) -> (continuation of Deferred@1288604902 after retry RPC@257302969) -> (continuation of Deferred@391011632 after retry RPC@690744713) -> (continuation of Deferred@1578138435 after retry RPC@673284781))) after lookup
20:03:46.751 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:46.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:46.752 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.016140  7440 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:03:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.235296  7456 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 8ba53d199a5c47b6acb057a49d2d0699)
20:03:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.235915  7456 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.238085  7456 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:47.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:47.243315  7121 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111)
20:03:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:47.247467  7121 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111)
20:03:47.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.252528  7393 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" is_pre_election: true
20:03:47.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.253402  7393 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e361d22f8464508ab46b5ee78fb2a3a in term 2.
20:03:47.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.255236  7123 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 8ba53d199a5c47b6acb057a49d2d0699
20:03:47.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.256208  7456 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 3 FOLLOWER]: Leader pre-election won for term 4
20:03:47.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.256764  7456 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 3 FOLLOWER]: Starting leader election (detected failure of leader 8ba53d199a5c47b6acb057a49d2d0699)
20:03:47.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.257154  7456 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 3 FOLLOWER]: Advancing to term 4
20:03:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.262490  7456 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.264273  7456 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 4 election: Requested vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:47.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.265502  7393 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4"
20:03:47.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.266083  7393 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 2 FOLLOWER]: Advancing to term 4
20:03:47.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:47.268265  7121 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111)
20:03:47.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.272838  7393 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e361d22f8464508ab46b5ee78fb2a3a in term 4.
20:03:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.273914  7123 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 8ba53d199a5c47b6acb057a49d2d0699
20:03:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.274571  7456 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 4 FOLLOWER]: Leader election won for term 4
20:03:47.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.275715  7456 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 4 LEADER]: Becoming Leader. State: Replica: 9e361d22f8464508ab46b5ee78fb2a3a, State: Running, Role: LEADER
20:03:47.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.277104  7456 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:47.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.286834  6475 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a reported cstate change: term changed from 3 to 4, leader changed from 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194) to 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195). New cstate: current_term: 4 leader_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: HEALTHY } } }
20:03:47.609 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:47.610 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:47.610 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1582), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [21ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [22ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [24ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [24ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [42ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [46ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [46ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [81ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [84ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [85ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [85ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [196ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [201ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [203ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [204ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [204ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [216ms] refreshing cache from master, [216ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [220ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [222ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [223ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [223ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [716ms] refreshing cache from master, [717ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [721ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [722ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [724ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [724ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [1577ms] refreshing cache from master, [1577ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [1581ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@83709950(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@3392b1bd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985) -> (continuation of Deferred@1288604902 after retry RPC@257302969) -> (continuation of Deferred@391011632 after retry RPC@690744713) -> (continuation of Deferred@1578138435 after retry RPC@673284781) -> (continuation of Deferred@1175992186 after retry RPC@1654026021), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3bb02910 -> wakeup thread Time-limited test -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985) -> (continuation of Deferred@1288604902 after retry RPC@257302969) -> (continuation of Deferred@391011632 after retry RPC@690744713) -> (continuation of Deferred@1578138435 after retry RPC@673284781) -> (continuation of Deferred@1175992186 after retry RPC@1654026021))) after lookup
20:03:47.612 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:47.613 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] Successfully connected to peer
20:03:47.641 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x0f70c03a, L:/127.0.0.1:46568 - R:/127.6.32.195:39759] using TOKEN/null
20:03:47.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.652609  7393 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 4 FOLLOWER]: Refusing update from remote peer 9e361d22f8464508ab46b5ee78fb2a3a: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
20:03:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.655236  7461 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.001s
20:03:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:47.656507  7121 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a -> Peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Couldn't send request to peer 8ba53d199a5c47b6acb057a49d2d0699. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:47.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.674715  7477 mvcc.cc:204] Tried to move back new op lower bound from 7305384032123699200 to 7305384017850388480. Current Snapshot: MvccSnapshot[applied={T|T < 7305384032123699200}]
20:03:47.730 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.32.194:41701
20:03:47.731 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:47.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:47.733320  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:47.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:47.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:03:47.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/logs
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.194:41701
20:03:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.194
20:03:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33139
20:03:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.194
20:03:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:48.099236  7484 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:48.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:48.099869  7484 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:48.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:48.100664  7484 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:48.144971  7484 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:48.145805  7484 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.194
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:48.185145  7484 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.194:41701
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.194
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33139
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.194
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/logs
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:48.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:48.186463  7484 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:48.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:48.188241  7484 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:48.201152  7490 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:49.606573  7489 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 7484
20:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:49.634734  7489 kernel_stack_watchdog.cc:198] Thread 7484 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
20:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:49.635362  7484 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.434s	user 0.443s	sys 0.891s
20:03:49.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:49.635804  7484 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.435s	user 0.443s	sys 0.891s
20:03:49.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:48.201777  7491 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:49.637055  7492 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1435 milliseconds
20:03:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:49.637992  7493 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.637993  7484 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:49.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.641508  7484 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:49.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.643826  7484 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:49.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.645200  7484 hybrid_clock.cc:648] HybridClock initialized: now 1783541029645155 us; error 67 us; skew 500 ppm
20:03:49.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.646078  7484 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:49.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.652598  7484 webserver.cc:533] Webserver started at http://127.6.32.194:33139/ using document root <none> and password file <none>
20:03:49.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.653692  7484 fs_manager.cc:362] Metadata directory not provided
20:03:49.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.654086  7484 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.662973  7484 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.001s
20:03:49.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.667863  7505 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:49.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.669193  7484 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
20:03:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.669509  7484 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:03:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:03:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:28 on dist-test-slave-xfg5"
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.671548  7484 fs_report.cc:389] FS layout report
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/data
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.736338  7484 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:49.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.738024  7484 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:49.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.738474  7484 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:49.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.741359  7484 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:49.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.747776  7512 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:49.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.757234  7484 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:49.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.757551  7484 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
20:03:49.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.757874  7484 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:49.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.763988  7484 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:49.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.764214  7484 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.006s	sys 0.000s
20:03:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.764937  7512 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Bootstrap starting.
20:03:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.835157  7512 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Log is configured to *not* fsync() on all Append() calls
20:03:49.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.891852  7512 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.892795  7512 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Bootstrap complete.
20:03:49.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.894368  7512 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Time spent bootstrapping tablet: real 0.130s	user 0.082s	sys 0.041s
20:03:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.920615  7512 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.924045  7512 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ba53d199a5c47b6acb057a49d2d0699, State: Initialized, Role: FOLLOWER
20:03:49.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.925139  7512 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.931070  7512 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Time spent starting tablet: real 0.036s	user 0.031s	sys 0.007s
20:03:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.974355  7484 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.194:41701
20:03:49.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.974548  7622 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.194:41701 every 8 connection(s)
20:03:49.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.978272  7484 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:03:49.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:49.983740  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 7484
20:03:49.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:50.021 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.050419  7624 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:03:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.051007  7624 heartbeater.cc:461] Registering TS with master...
20:03:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.052404  7624 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:03:50.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.057662  6475 ts_manager.cc:194] Re-registered known tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:03:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.061267  6475 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.194:44521
20:03:50.062 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.32.195:39759
20:03:50.062 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:50.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.062914  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 7093
20:03:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.074200  7623 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:03:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.074613  7623 heartbeater.cc:461] Registering TS with master...
20:03:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.075430  7623 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:03:50.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.080088  6396 ts_manager.cc:194] Re-registered known tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:03:50.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.087198  7625 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:03:50.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.087616  7625 heartbeater.cc:461] Registering TS with master...
20:03:50.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.088269  7625 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:03:50.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:50.092134  6321 ts_manager.cc:194] Re-registered known tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:03:50.128 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] handling channelInactive
20:03:50.128 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:50.128 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state READY due to: connection closed
20:03:50.129 [DEBUG - Time-limited test] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:50.130 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.130 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.130 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.148 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:50.149 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:50.149 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1205945865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@165fe697 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@738c590b -> wakeup thread Time-limited test)) after lookup
20:03:50.150 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:50.151 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.151 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.151 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.170 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:50.170 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:50.171 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [21ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [23ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [41ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1205945865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@165fe697 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1064030193 after retry RPC@784243990), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@738c590b -> wakeup thread Time-limited test -> (continuation of Deferred@1064030193 after retry RPC@784243990))) after lookup
20:03:50.172 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:50.173 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.173 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.173 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.189 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:50.189 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:50.190 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=61), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [21ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [23ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [41ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [43ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [45ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [45ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [60ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1205945865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@165fe697 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1064030193 after retry RPC@784243990) -> (continuation of Deferred@1688150948 after retry RPC@1041454092), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@738c590b -> wakeup thread Time-limited test -> (continuation of Deferred@1064030193 after retry RPC@784243990) -> (continuation of Deferred@1688150948 after retry RPC@1041454092))) after lookup
20:03:50.190 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:50.191 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.191 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.192 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.229 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:50.229 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:50.229 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=101), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [21ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [23ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [41ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [43ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [45ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [45ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [60ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [62ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [63ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [64ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [100ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1205945865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@165fe697 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1064030193 after retry RPC@784243990) -> (continuation of Deferred@1688150948 after retry RPC@1041454092) -> (continuation of Deferred@133355349 after retry RPC@593838227), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@738c590b -> wakeup thread Time-limited test -> (continuation of Deferred@1064030193 after retry RPC@784243990) -> (continuation of Deferred@1688150948 after retry RPC@1041454092) -> (continuation of Deferred@133355349 after retry RPC@593838227))) after lookup
20:03:50.230 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:50.231 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.231 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.232 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.671 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:50.671 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:03:50.672 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=543), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [21ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [23ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [41ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [43ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [45ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [45ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [60ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [62ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [63ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [64ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [100ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [102ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [104ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [104ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [536ms] refreshing cache from master, [537ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [542ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1205945865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@165fe697 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1064030193 after retry RPC@784243990) -> (continuation of Deferred@1688150948 after retry RPC@1041454092) -> (continuation of Deferred@133355349 after retry RPC@593838227) -> (continuation of Deferred@736700408 after retry RPC@570526670), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@738c590b -> wakeup thread Time-limited test -> (continuation of Deferred@1064030193 after retry RPC@784243990) -> (continuation of Deferred@1688150948 after retry RPC@1041454092) -> (continuation of Deferred@133355349 after retry RPC@593838227) -> (continuation of Deferred@736700408 after retry RPC@570526670))) after lookup
20:03:50.673 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:03:50.674 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.674 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:50.675 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:03:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.069090  7624 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:03:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.290701  7635 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.291188  7635 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:51.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.293503  7635 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:51.300444  7509 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:03:51.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:51.304996  7509 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:03:51.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.306514  7393 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "8ba53d199a5c47b6acb057a49d2d0699" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" is_pre_election: true
20:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.308022  7509 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8ba53d199a5c47b6acb057a49d2d0699; no voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 9e361d22f8464508ab46b5ee78fb2a3a
20:03:51.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.308832  7635 raft_consensus.cc:2749] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
20:03:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.329937  7640 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 9e361d22f8464508ab46b5ee78fb2a3a)
20:03:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.330415  7640 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:51.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.332935  7640 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:51.337248  7327 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:03:51.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:51.342064  7327 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:03:51.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.346743  7577 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699" is_pre_election: true
20:03:51.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.347433  7577 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a9f83cc3ec74be2b5c075c8011c00f4 in term 3.
20:03:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.348966  7326 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 8ba53d199a5c47b6acb057a49d2d0699; no voters: 9e361d22f8464508ab46b5ee78fb2a3a
20:03:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.349653  7640 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 4 FOLLOWER]: Leader pre-election won for term 5
20:03:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.349990  7640 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 9e361d22f8464508ab46b5ee78fb2a3a)
20:03:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.350409  7640 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 4 FOLLOWER]: Advancing to term 5
20:03:51.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.356370  7640 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:51.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.358116  7640 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 5 election: Requested vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.359129  7577 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:03:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.359733  7577 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 3 FOLLOWER]: Advancing to term 5
20:03:51.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:51.363523  7327 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:03:51.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.367484  7577 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a9f83cc3ec74be2b5c075c8011c00f4 in term 5.
20:03:51.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.368664  7326 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 8ba53d199a5c47b6acb057a49d2d0699; no voters: 9e361d22f8464508ab46b5ee78fb2a3a
20:03:51.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.369419  7640 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 5 FOLLOWER]: Leader election won for term 5
20:03:51.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.371042  7640 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 5 LEADER]: Becoming Leader. State: Replica: 1a9f83cc3ec74be2b5c075c8011c00f4, State: Running, Role: LEADER
20:03:51.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.371914  7640 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:51.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.382616  6475 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 reported cstate change: term changed from 4 to 5, leader changed from 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195) to 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193). New cstate: current_term: 5 leader_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: UNKNOWN } } }
20:03:51.529 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:51.529 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:51.529 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1401), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [21ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [23ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [23ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [41ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [43ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [45ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [45ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [60ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [62ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [63ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [64ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [100ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [102ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [104ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [104ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [536ms] refreshing cache from master, [537ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [542ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [544ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [546ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [547ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [1396ms] refreshing cache from master, [1396ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [1400ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1205945865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@165fe697 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1064030193 after retry RPC@784243990) -> (continuation of Deferred@1688150948 after retry RPC@1041454092) -> (continuation of Deferred@133355349 after retry RPC@593838227) -> (continuation of Deferred@736700408 after retry RPC@570526670) -> (continuation of Deferred@860933105 after retry RPC@488729006), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@738c590b -> wakeup thread Time-limited test -> (continuation of Deferred@1064030193 after retry RPC@784243990) -> (continuation of Deferred@1688150948 after retry RPC@1041454092) -> (continuation of Deferred@133355349 after retry RPC@593838227) -> (continuation of Deferred@736700408 after retry RPC@570526670) -> (continuation of Deferred@860933105 after retry RPC@488729006))) after lookup
20:03:51.530 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:51.532 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] Successfully connected to peer
20:03:51.549 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x4dc3e419, L:/127.0.0.1:36998 - R:/127.6.32.193:39013] using TOKEN/null
20:03:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.558636  7577 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 5 FOLLOWER]: Refusing update from remote peer 1a9f83cc3ec74be2b5c075c8011c00f4: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 18. (index mismatch)
20:03:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.560916  7645 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.001s
20:03:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:51.562080  7327 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 -> Peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Couldn't send request to peer 9e361d22f8464508ab46b5ee78fb2a3a. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:51.627 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.32.195:39759
20:03:51.627 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:51.629182  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/logs
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.195:39759
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.195
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36303
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.195
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:51.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:51.964973  7664 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:51.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:51.965535  7664 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:51.966205  7664 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:52.026891  7664 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:52.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:52.028340  7664 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.195
20:03:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:52.071103  7664 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.195:39759
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.195
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36303
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.195
20:03:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/logs
20:03:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:52.072922  7664 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:52.075431  7664 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:52.124508  7671 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:53.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:52.139114  7673 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:53.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:53.481189  7664 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.354s	user 0.559s	sys 0.787s
20:03:53.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:53.481521  7664 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.355s	user 0.559s	sys 0.787s
20:03:53.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.483784  7664 server_base.cc:1061] running on GCE node
20:03:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:53.485751  7682 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.487176  7664 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.489778  7664 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.491219  7664 hybrid_clock.cc:648] HybridClock initialized: now 1783541033491169 us; error 71 us; skew 500 ppm
20:03:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.492182  7664 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:53.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.500801  7664 webserver.cc:533] Webserver started at http://127.6.32.195:36303/ using document root <none> and password file <none>
20:03:53.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.501971  7664 fs_manager.cc:362] Metadata directory not provided
20:03:53.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.502306  7664 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:53.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.512378  7664 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.006s	sys 0.002s
20:03:53.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.518960  7687 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:53.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.520498  7664 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.001s
20:03:53.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.520983  7664 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:53.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e361d22f8464508ab46b5ee78fb2a3a"
20:03:53.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:31 on dist-test-slave-xfg5"
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.524252  7664 fs_report.cc:389] FS layout report
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/data
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.614118  7664 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.616639  7664 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.617291  7664 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.621848  7664 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:53.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.630394  7694 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:53.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.639632  7664 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:53.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.639879  7664 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.001s	sys 0.001s
20:03:53.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.640131  7664 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.645777  7664 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.645984  7664 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.003s	sys 0.000s
20:03:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.646652  7694 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap starting.
20:03:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.716239  7694 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Log is configured to *not* fsync() on all Append() calls
20:03:53.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.783383  7694 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.784766  7694 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap complete.
20:03:53.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.786901  7694 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Time spent bootstrapping tablet: real 0.141s	user 0.081s	sys 0.057s
20:03:53.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.812256  7694 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:53.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.814881  7694 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e361d22f8464508ab46b5ee78fb2a3a, State: Initialized, Role: FOLLOWER
20:03:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.816342  7694 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:53.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.821337  7694 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Time spent starting tablet: real 0.034s	user 0.029s	sys 0.004s
20:03:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.856596  7664 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.195:39759
20:03:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.857043  7805 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.195:39759 every 8 connection(s)
20:03:53.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.859560  7664 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:03:53.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.869453  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 7664
20:03:53.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:53.916 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:53.918 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.32.193:39013
20:03:53.919 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.916623  7806 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:03:53.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.917562  7806 heartbeater.cc:461] Registering TS with master...
20:03:53.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.919083  7806 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:03:53.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.920181  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 7300
20:03:53.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.924692  7808 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:03:53.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.925268  7808 heartbeater.cc:461] Registering TS with master...
20:03:53.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.925647  6396 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:53.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.926270  7808 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:03:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.929461  7807 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:03:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.929809  7807 heartbeater.cc:461] Registering TS with master...
20:03:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.929862  6321 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:53.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.930800  7807 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:03:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.933578  6475 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:53.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:53.937008  6475 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.195:36307
20:03:53.954 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] handling channelInactive
20:03:53.954 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:53.954 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state READY due to: connection closed
20:03:53.955 [DEBUG - Time-limited test] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:53.956 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:53.956 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:53.956 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:53.968 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:53.969 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:53.969 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1667069452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@66fa7684 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26684f34 -> wakeup thread Time-limited test)) after lookup
20:03:53.970 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:53.972 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:53.972 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:53.972 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:53.989 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:53.989 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:53.990 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [16ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [18ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [18ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1667069452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@66fa7684 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@414159496 after retry RPC@2086301510), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26684f34 -> wakeup thread Time-limited test -> (continuation of Deferred@414159496 after retry RPC@2086301510))) after lookup
20:03:53.991 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:53.993 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:53.993 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:53.993 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:54.029 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:54.029 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:54.030 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [16ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [18ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [18ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [36ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [39ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [39ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [75ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1667069452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@66fa7684 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@414159496 after retry RPC@2086301510) -> (continuation of Deferred@1069162384 after retry RPC@280323966), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26684f34 -> wakeup thread Time-limited test -> (continuation of Deferred@414159496 after retry RPC@2086301510) -> (continuation of Deferred@1069162384 after retry RPC@280323966))) after lookup
20:03:54.030 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:54.031 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:54.032 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:54.032 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:54.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:54.109 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:54.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=155), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [16ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [18ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [18ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [36ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [39ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [39ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [75ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [76ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [78ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [78ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1667069452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@66fa7684 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@414159496 after retry RPC@2086301510) -> (continuation of Deferred@1069162384 after retry RPC@280323966) -> (continuation of Deferred@1733814105 after retry RPC@1549988442), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26684f34 -> wakeup thread Time-limited test -> (continuation of Deferred@414159496 after retry RPC@2086301510) -> (continuation of Deferred@1069162384 after retry RPC@280323966) -> (continuation of Deferred@1733814105 after retry RPC@1549988442))) after lookup
20:03:54.111 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:54.112 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:54.112 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:54.112 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:54.168 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:54.169 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:54.169 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=214), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [16ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [18ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [18ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [36ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [39ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [39ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [75ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [76ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [78ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [78ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [156ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [158ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [158ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [210ms] refreshing cache from master, [210ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [214ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1667069452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@66fa7684 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@414159496 after retry RPC@2086301510) -> (continuation of Deferred@1069162384 after retry RPC@280323966) -> (continuation of Deferred@1733814105 after retry RPC@1549988442) -> (continuation of Deferred@317590006 after retry RPC@1507183059), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26684f34 -> wakeup thread Time-limited test -> (continuation of Deferred@414159496 after retry RPC@2086301510) -> (continuation of Deferred@1069162384 after retry RPC@280323966) -> (continuation of Deferred@1733814105 after retry RPC@1549988442) -> (continuation of Deferred@317590006 after retry RPC@1507183059))) after lookup
20:03:54.170 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:03:54.171 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:54.171 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:54.172 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:03:54.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:54.940574  7807 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:03:55.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.053035  7817 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 1a9f83cc3ec74be2b5c075c8011c00f4)
20:03:55.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.053601  7817 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.055917  7817 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:55.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:55.075250  7509 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111)
20:03:55.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.084550  7757 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "8ba53d199a5c47b6acb057a49d2d0699" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" is_pre_election: true
20:03:55.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.085397  7757 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8ba53d199a5c47b6acb057a49d2d0699 in term 4.
20:03:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.086922  7509 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8ba53d199a5c47b6acb057a49d2d0699, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 1a9f83cc3ec74be2b5c075c8011c00f4
20:03:55.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.087738  7817 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 5 FOLLOWER]: Leader pre-election won for term 6
20:03:55.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.088075  7817 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 1a9f83cc3ec74be2b5c075c8011c00f4)
20:03:55.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.088392  7817 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 5 FOLLOWER]: Advancing to term 6
20:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.095515  7817 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:55.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.097455  7817 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 6 election: Requested vote from peers 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:03:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.100644  7757 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "8ba53d199a5c47b6acb057a49d2d0699" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "9e361d22f8464508ab46b5ee78fb2a3a"
20:03:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.101212  7757 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 4 FOLLOWER]: Advancing to term 6
20:03:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:55.102702  7509 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111)
20:03:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.111217  7757 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8ba53d199a5c47b6acb057a49d2d0699 in term 6.
20:03:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.112671  7509 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8ba53d199a5c47b6acb057a49d2d0699, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 1a9f83cc3ec74be2b5c075c8011c00f4
20:03:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.113740  7817 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 6 FOLLOWER]: Leader election won for term 6
20:03:55.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.115614  7817 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 6 LEADER]: Becoming Leader. State: Replica: 8ba53d199a5c47b6acb057a49d2d0699, State: Running, Role: LEADER
20:03:55.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.117031  7817 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.127617  6475 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 reported cstate change: term changed from 5 to 6, leader changed from 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193) to 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194). New cstate: current_term: 6 leader_uuid: "8ba53d199a5c47b6acb057a49d2d0699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: UNKNOWN } } }
20:03:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.622642  7757 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 6 FOLLOWER]: Refusing update from remote peer 8ba53d199a5c47b6acb057a49d2d0699: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
20:03:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:55.624241  7817 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
20:03:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:55.672827  7509 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 -> Peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Couldn't send request to peer 1a9f83cc3ec74be2b5c075c8011c00f4. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:55.950 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:55.950 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:55.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1996), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [16ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [18ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [18ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [36ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [39ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [39ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [75ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [76ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [78ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [78ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [156ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [158ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [158ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [210ms] refreshing cache from master, [210ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [214ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [215ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [218ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [218ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [1990ms] refreshing cache from master, [1991ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [1995ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1667069452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@66fa7684 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@414159496 after retry RPC@2086301510) -> (continuation of Deferred@1069162384 after retry RPC@280323966) -> (continuation of Deferred@1733814105 after retry RPC@1549988442) -> (continuation of Deferred@317590006 after retry RPC@1507183059) -> (continuation of Deferred@1314701861 after retry RPC@553469906), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26684f34 -> wakeup thread Time-limited test -> (continuation of Deferred@414159496 after retry RPC@2086301510) -> (continuation of Deferred@1069162384 after retry RPC@280323966) -> (continuation of Deferred@1733814105 after retry RPC@1549988442) -> (continuation of Deferred@317590006 after retry RPC@1507183059) -> (continuation of Deferred@1314701861 after retry RPC@553469906))) after lookup
20:03:55.952 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:55.953 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] Successfully connected to peer
20:03:55.996 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0x458b646f, L:/127.0.0.1:50792 - R:/127.6.32.194:41701] using TOKEN/null
20:03:56.066 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.32.193:39013
20:03:56.067 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:56.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:56.069343  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:56.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/logs
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.193:39013
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.193
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39967
20:03:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.193
20:03:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:56.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:56.432686  7848 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:56.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:56.433218  7848 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:56.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:56.433678  7848 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:56.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:56.469290  7848 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:56.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:56.470167  7848 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.193
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:56.510809  7848 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.193:39013
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.193
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39967
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.193
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/logs
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:56.512295  7848 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:56.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:56.513969  7848 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:56.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:56.528051  7855 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:56.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:56.582860  7509 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111) [suppressed 4 similar messages]
20:03:57.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:57.922245  7509 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 -> Peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Couldn't send request to peer 1a9f83cc3ec74be2b5c075c8011c00f4. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:57.932621  7854 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 7848
20:03:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:58.290423  7854 kernel_stack_watchdog.cc:198] Thread 7848 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:03:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:03:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:03:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:03:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:03:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:56.528703  7856 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:58.291327  7848 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.764s	user 0.638s	sys 1.028s
20:03:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:58.292093  7848 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.765s	user 0.638s	sys 1.028s
20:03:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:58.293763  7858 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:58.296761  7857 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1765 milliseconds
20:03:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.296846  7848 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.298043  7848 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:58.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.300135  7848 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:58.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.301527  7848 hybrid_clock.cc:648] HybridClock initialized: now 1783541038301485 us; error 78 us; skew 500 ppm
20:03:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.302335  7848 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:58.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.310608  7848 webserver.cc:533] Webserver started at http://127.6.32.193:39967/ using document root <none> and password file <none>
20:03:58.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.311671  7848 fs_manager.cc:362] Metadata directory not provided
20:03:58.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.311903  7848 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.319927  7848 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.002s
20:03:58.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.324678  7867 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:58.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.325968  7848 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
20:03:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.326287  7848 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a9f83cc3ec74be2b5c075c8011c00f4"
20:03:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:26 on dist-test-slave-xfg5"
20:03:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.328362  7848 fs_report.cc:389] FS layout report
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/data
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.388700  7848 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:58.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.390760  7848 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:58.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.391340  7848 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:58.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.394289  7848 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.400178  7874 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:58.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.408903  7848 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:58.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.409162  7848 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
20:03:58.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.409385  7848 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.414413  7848 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.414631  7848 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
20:03:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.415459  7874 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Bootstrap starting.
20:03:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.481756  7874 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Log is configured to *not* fsync() on all Append() calls
20:03:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.565527  7874 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.567092  7874 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Bootstrap complete.
20:03:58.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.568578  7874 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Time spent bootstrapping tablet: real 0.154s	user 0.120s	sys 0.028s
20:03:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.587601  7874 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.590781  7874 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a9f83cc3ec74be2b5c075c8011c00f4, State: Initialized, Role: FOLLOWER
20:03:58.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.591939  7874 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.596298  7874 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Time spent starting tablet: real 0.027s	user 0.024s	sys 0.004s
20:03:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.605898  7848 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.193:39013
20:03:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.606328  7983 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.193:39013 every 8 connection(s)
20:03:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.608758  7848 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:03:58.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.615756  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 7848
20:03:58.617 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:58.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.670125  7986 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:03:58.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.670931  7986 heartbeater.cc:461] Registering TS with master...
20:03:58.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.672827  7986 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:03:58.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.675318  7984 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:03:58.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.675711  7984 heartbeater.cc:461] Registering TS with master...
20:03:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.676738  7984 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:03:58.680 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:58.680 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.32.194:41701
20:03:58.681 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.681789  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 7484
20:03:58.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.683935  7985 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:03:58.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.684356  7985 heartbeater.cc:461] Registering TS with master...
20:03:58.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.685302  7985 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:03:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.686308  6320 ts_manager.cc:194] Re-registered known tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.688462  6475 ts_manager.cc:194] Re-registered known tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.690266  6396 ts_manager.cc:194] Re-registered known tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:58.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:58.691644  6475 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.193:60549
20:03:58.714 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] handling channelInactive
20:03:58.715 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state READY due to: connection closed
20:03:58.715 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:58.716 [DEBUG - Time-limited test] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:58.717 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.717 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.718 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.729 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:58.729 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:58.729 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@609164854(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@5fb66faf -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2120ca08 -> wakeup thread Time-limited test)) after lookup
20:03:58.731 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:58.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.733 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.733 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.748 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:58.748 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:58.749 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [14ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [32ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@609164854(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@5fb66faf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2120ca08 -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810))) after lookup
20:03:58.749 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:58.751 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.751 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.751 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.788 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:58.788 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:58.789 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [14ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [32ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [33ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [35ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [35ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@609164854(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@5fb66faf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2120ca08 -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176))) after lookup
20:03:58.789 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:58.791 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.791 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.791 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.909 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:58.909 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:58.909 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=193), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [14ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [32ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [33ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [35ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [35ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [73ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [75ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [75ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [192ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@609164854(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@5fb66faf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176) -> (continuation of Deferred@1253861865 after retry RPC@1809887435), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2120ca08 -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176) -> (continuation of Deferred@1253861865 after retry RPC@1809887435))) after lookup
20:03:58.910 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:58.911 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.912 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.912 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.989 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:58.989 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:58.989 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=273), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [14ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [32ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [33ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [35ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [35ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [73ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [75ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [75ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [192ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [194ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [196ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [196ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [268ms] refreshing cache from master, [268ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [272ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@609164854(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@5fb66faf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176) -> (continuation of Deferred@1253861865 after retry RPC@1809887435) -> (continuation of Deferred@156445416 after retry RPC@1742452193), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2120ca08 -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176) -> (continuation of Deferred@1253861865 after retry RPC@1809887435) -> (continuation of Deferred@156445416 after retry RPC@1742452193))) after lookup
20:03:58.990 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:58.992 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.992 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:58.992 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.615788  7995 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 8ba53d199a5c47b6acb057a49d2d0699)
20:03:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.616302  7995 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:59.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.619159  7995 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:59.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:59.623636  7690 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111)
20:03:59.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:59.638492  7690 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111)
20:03:59.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.639461  7938 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" is_pre_election: true
20:03:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.640132  7938 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e361d22f8464508ab46b5ee78fb2a3a in term 5.
20:03:59.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.641465  7691 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 8ba53d199a5c47b6acb057a49d2d0699
20:03:59.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.642181  7995 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 6 FOLLOWER]: Leader pre-election won for term 7
20:03:59.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.642501  7995 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 6 FOLLOWER]: Starting leader election (detected failure of leader 8ba53d199a5c47b6acb057a49d2d0699)
20:03:59.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.642818  7995 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 6 FOLLOWER]: Advancing to term 7
20:03:59.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.647634  7995 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:59.649 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:03:59.649 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:03:59.649 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=933), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [14ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [32ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [33ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [35ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [35ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [73ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [75ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [75ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [192ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [194ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [196ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [196ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [268ms] refreshing cache from master, [268ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [272ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [274ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [276ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [276ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [928ms] refreshing cache from master, [929ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [932ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@609164854(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@5fb66faf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176) -> (continuation of Deferred@1253861865 after retry RPC@1809887435) -> (continuation of Deferred@156445416 after retry RPC@1742452193) -> (continuation of Deferred@1558475324 after retry RPC@1714069756), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2120ca08 -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176) -> (continuation of Deferred@1253861865 after retry RPC@1809887435) -> (continuation of Deferred@156445416 after retry RPC@1742452193) -> (continuation of Deferred@1558475324 after retry RPC@1714069756))) after lookup
20:03:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.649593  7995 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 7 election: Requested vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:03:59.650 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:03:59.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.651307  7938 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4"
20:03:59.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.651917  7938 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 5 FOLLOWER]: Advancing to term 7
20:03:59.653 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:59.653 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:59.653 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:03:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:03:59.654764  7690 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111)
20:03:59.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.659780  7938 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e361d22f8464508ab46b5ee78fb2a3a in term 7.
20:03:59.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.660802  7691 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 8ba53d199a5c47b6acb057a49d2d0699
20:03:59.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.661458  7995 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 7 FOLLOWER]: Leader election won for term 7
20:03:59.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.663075  7995 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 7 LEADER]: Becoming Leader. State: Replica: 9e361d22f8464508ab46b5ee78fb2a3a, State: Running, Role: LEADER
20:03:59.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.664445  7995 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:03:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.673931  6475 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a reported cstate change: term changed from 6 to 7, leader changed from 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194) to 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195). New cstate: current_term: 7 leader_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: HEALTHY } } }
20:03:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:03:59.695374  7985 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:04:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:00.240782  7690 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a -> Peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Couldn't send request to peer 8ba53d199a5c47b6acb057a49d2d0699. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:00.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:00.286362  7938 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 7 FOLLOWER]: Refusing update from remote peer 9e361d22f8464508ab46b5ee78fb2a3a: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
20:04:00.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:00.288509  8013 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
20:04:00.590 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:00.590 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:00.591 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1875), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [2ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [14ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [32ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [33ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [35ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [35ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [73ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [75ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [75ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [192ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [194ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [196ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [196ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [268ms] refreshing cache from master, [268ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [272ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [274ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [276ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [276ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [928ms] refreshing cache from master, [929ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [932ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [934ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [937ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [937ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [1868ms] refreshing cache from master, [1868ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [1874ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@609164854(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@5fb66faf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176) -> (continuation of Deferred@1253861865 after retry RPC@1809887435) -> (continuation of Deferred@156445416 after retry RPC@1742452193) -> (continuation of Deferred@1558475324 after retry RPC@1714069756) -> (continuation of Deferred@532740997 after retry RPC@2064805427), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2120ca08 -> wakeup thread Time-limited test -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176) -> (continuation of Deferred@1253861865 after retry RPC@1809887435) -> (continuation of Deferred@156445416 after retry RPC@1742452193) -> (continuation of Deferred@1558475324 after retry RPC@1714069756) -> (continuation of Deferred@532740997 after retry RPC@2064805427))) after lookup
20:04:00.592 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:00.593 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] Successfully connected to peer
20:04:00.620 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0x4af55e8c, L:/127.0.0.1:54546 - R:/127.6.32.195:39759] using TOKEN/null
20:04:00.671 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.32.194:41701
20:04:00.671 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:00.673278  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/logs
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.194:41701
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.194
20:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:04:00.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33139
20:04:00.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.194
20:04:00.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:04:00.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:00.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:00.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:01.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:01.012466  8030 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:01.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:01.013055  8030 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:01.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:01.013674  8030 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:01.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:01.050611  8030 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:01.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:01.051733  8030 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.194
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:01.099035  8030 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.194:41701
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.194
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33139
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.194
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/logs
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:04:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:01.100713  8030 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:01.102859  8030 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:04:01.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:01.120775  8038 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:02.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:01.121801  8039 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:02.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:02.462018  8041 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:02.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:02.466539  8040 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1341 milliseconds
20:04:02.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.466627  8030 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:02.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.468015  8030 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.470863  8030 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.472342  8030 hybrid_clock.cc:648] HybridClock initialized: now 1783541042472289 us; error 73 us; skew 500 ppm
20:04:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.473151  8030 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.481204  8030 webserver.cc:533] Webserver started at http://127.6.32.194:33139/ using document root <none> and password file <none>
20:04:02.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.482134  8030 fs_manager.cc:362] Metadata directory not provided
20:04:02.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.482332  8030 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:02.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.491132  8030 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.003s	sys 0.003s
20:04:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.496996  8053 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.498369  8030 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.006s	sys 0.000s
20:04:02.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.498796  8030 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:28 on dist-test-slave-xfg5"
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.500828  8030 fs_report.cc:389] FS layout report
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/data
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:02.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.580547  8030 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:02.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.582453  8030 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:02.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.583140  8030 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.586340  8030 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.593156  8060 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:04:02.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.601838  8030 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:02.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.602171  8030 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
20:04:02.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.602517  8030 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:02.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.608212  8030 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:02.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.608484  8030 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.005s	sys 0.000s
20:04:02.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.608982  8060 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Bootstrap starting.
20:04:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.682278  8060 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Log is configured to *not* fsync() on all Append() calls
20:04:02.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:02.761982  7690 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a -> Peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Couldn't send request to peer 8ba53d199a5c47b6acb057a49d2d0699. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:04:02.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.774459  8060 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:02.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.775538  8060 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Bootstrap complete.
20:04:02.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.777343  8060 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Time spent bootstrapping tablet: real 0.169s	user 0.119s	sys 0.044s
20:04:02.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.796940  8060 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.799784  8060 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ba53d199a5c47b6acb057a49d2d0699, State: Initialized, Role: FOLLOWER
20:04:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.800953  8060 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:02.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.804949  8060 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Time spent starting tablet: real 0.027s	user 0.024s	sys 0.004s
20:04:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.813606  8030 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.194:41701
20:04:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.813941  8173 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.194:41701 every 8 connection(s)
20:04:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.816422  8030 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:04:02.821 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.819886  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 8030
20:04:02.864 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:02.866 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.32.195:39759
20:04:02.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:04:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.867215  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 7664
20:04:02.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.878067  8174 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:04:02.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.878826  8174 heartbeater.cc:461] Registering TS with master...
20:04:02.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.880016  8175 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:04:02.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.880352  8175 heartbeater.cc:461] Registering TS with master...
20:04:02.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.880625  8174 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:04:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.881520  8175 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:04:02.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.886920  6396 ts_manager.cc:194] Re-registered known tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:04:02.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.891867  8176 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:04:02.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.892289  8176 heartbeater.cc:461] Registering TS with master...
20:04:02.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.892448  6474 ts_manager.cc:194] Re-registered known tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:04:02.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.893304  8176 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:04:02.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.895219  6474 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.194:52583
20:04:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:02.896919  6320 ts_manager.cc:194] Re-registered known tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:04:02.910 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] handling channelInactive
20:04:02.910 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state READY due to: connection closed
20:04:02.911 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:02.912 [DEBUG - Time-limited test] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:02.913 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.913 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.930 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:02.930 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:02.930 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@704661241(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@61b71e0a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14aac985 -> wakeup thread Time-limited test)) after lookup
20:04:02.931 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:02.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.932 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.950 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:02.950 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:02.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [21ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [21ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@704661241(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@61b71e0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14aac985 -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727))) after lookup
20:04:02.952 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:02.953 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.953 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.953 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.989 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:02.989 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:02.990 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=78), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [21ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [21ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [41ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [42ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [42ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [78ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@704661241(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@61b71e0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727) -> (continuation of Deferred@989299333 after retry RPC@1375518510), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14aac985 -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727) -> (continuation of Deferred@989299333 after retry RPC@1375518510))) after lookup
20:04:02.991 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:02.992 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:02.992 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:03.128 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:03.129 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:03.129 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=217), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [21ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [21ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [41ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [42ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [42ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [78ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [79ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [81ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [81ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [213ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [217ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@704661241(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@61b71e0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727) -> (continuation of Deferred@989299333 after retry RPC@1375518510) -> (continuation of Deferred@1715641013 after retry RPC@1383292060), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14aac985 -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727) -> (continuation of Deferred@989299333 after retry RPC@1375518510) -> (continuation of Deferred@1715641013 after retry RPC@1383292060))) after lookup
20:04:03.130 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:03.131 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:03.131 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:03.131 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:03.449 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:03.450 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:03.450 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=538), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [21ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [21ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [41ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [42ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [42ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [78ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [79ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [81ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [81ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [213ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [217ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [219ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [220ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [221ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [533ms] refreshing cache from master, [533ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [538ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@704661241(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@61b71e0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727) -> (continuation of Deferred@989299333 after retry RPC@1375518510) -> (continuation of Deferred@1715641013 after retry RPC@1383292060) -> (continuation of Deferred@1900758167 after retry RPC@957641118), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14aac985 -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727) -> (continuation of Deferred@989299333 after retry RPC@1375518510) -> (continuation of Deferred@1715641013 after retry RPC@1383292060) -> (continuation of Deferred@1900758167 after retry RPC@957641118))) after lookup
20:04:03.450 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:03.451 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:03.452 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:03.452 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:03.771 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:03.771 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:03.772 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=860), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [21ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [21ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [41ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [42ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [42ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [78ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [79ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [81ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [81ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [213ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [217ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [219ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [220ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [221ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [533ms] refreshing cache from master, [533ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [538ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [539ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [541ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [541ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [853ms] refreshing cache from master, [853ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [859ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@704661241(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@61b71e0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727) -> (continuation of Deferred@989299333 after retry RPC@1375518510) -> (continuation of Deferred@1715641013 after retry RPC@1383292060) -> (continuation of Deferred@1900758167 after retry RPC@957641118) -> (continuation of Deferred@1742227718 after retry RPC@2111462626), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14aac985 -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727) -> (continuation of Deferred@989299333 after retry RPC@1375518510) -> (continuation of Deferred@1715641013 after retry RPC@1383292060) -> (continuation of Deferred@1900758167 after retry RPC@957641118) -> (continuation of Deferred@1742227718 after retry RPC@2111462626))) after lookup
20:04:03.772 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:03.774 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:03.774 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:03.774 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:03.899510  8175 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:04:04.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.193455  8185 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 9e361d22f8464508ab46b5ee78fb2a3a)
20:04:04.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.194013  8185 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:04.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.196516  8185 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:04.202451  7871 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:04:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:04.207154  7871 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:04:04.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.212667  8125 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699" is_pre_election: true
20:04:04.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.213563  8125 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a9f83cc3ec74be2b5c075c8011c00f4 in term 6.
20:04:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.215195  7870 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 8ba53d199a5c47b6acb057a49d2d0699; no voters: 9e361d22f8464508ab46b5ee78fb2a3a
20:04:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.215902  8185 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 7 FOLLOWER]: Leader pre-election won for term 8
20:04:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.216156  8185 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 9e361d22f8464508ab46b5ee78fb2a3a)
20:04:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.216377  8185 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 7 FOLLOWER]: Advancing to term 8
20:04:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.221388  8185 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:04.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.224131  8185 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 8 election: Requested vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:04.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.225270  8125 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:04:04.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.226173  8125 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 6 FOLLOWER]: Advancing to term 8
20:04:04.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:04.230195  7871 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:04:04.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.235354  8125 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a9f83cc3ec74be2b5c075c8011c00f4 in term 8.
20:04:04.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.236822  7870 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 8ba53d199a5c47b6acb057a49d2d0699; no voters: 9e361d22f8464508ab46b5ee78fb2a3a
20:04:04.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.237602  8185 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 8 FOLLOWER]: Leader election won for term 8
20:04:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.239586  8185 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 8 LEADER]: Becoming Leader. State: Replica: 1a9f83cc3ec74be2b5c075c8011c00f4, State: Running, Role: LEADER
20:04:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.240468  8185 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:04.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.252377  6474 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 reported cstate change: term changed from 7 to 8, leader changed from 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195) to 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193). New cstate: current_term: 8 leader_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: UNKNOWN } } }
20:04:04.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:04.705641  7871 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 -> Peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Couldn't send request to peer 9e361d22f8464508ab46b5ee78fb2a3a. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:04.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.854642  8125 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 8 FOLLOWER]: Refusing update from remote peer 1a9f83cc3ec74be2b5c075c8011c00f4: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
20:04:04.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:04.856508  8185 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
20:04:05.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:05.981129  6352 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5340 after lost signal to thread 6288
20:04:05.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:05.981835  6352 debug-util.cc:398] Leaking SignalData structure 0x7b080008ce40 after lost signal to thread 6355
20:04:06.970 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:06.970 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:06.971 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=4059), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [2ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [13ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [21ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [21ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [41ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [42ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [42ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [78ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [79ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [81ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [81ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [213ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [217ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [219ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [220ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [221ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [533ms] refreshing cache from master, [533ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [538ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [539ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [541ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [541ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [853ms] refreshing cache from master, [853ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [859ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [861ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [863ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [863ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [4054ms] refreshing cache from master, [4054ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [4058ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@704661241(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@61b71e0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727) -> (continuation of Deferred@989299333 after retry RPC@1375518510) -> (continuation of Deferred@1715641013 after retry RPC@1383292060) -> (continuation of Deferred@1900758167 after retry RPC@957641118) -> (continuation of Deferred@1742227718 after retry RPC@2111462626) -> (continuation of Deferred@597192725 after retry RPC@210372040), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@14aac985 -> wakeup thread Time-limited test -> (continuation of Deferred@609591745 after retry RPC@1727111727) -> (continuation of Deferred@989299333 after retry RPC@1375518510) -> (continuation of Deferred@1715641013 after retry RPC@1383292060) -> (continuation of Deferred@1900758167 after retry RPC@957641118) -> (continuation of Deferred@1742227718 after retry RPC@2111462626) -> (continuation of Deferred@597192725 after retry RPC@210372040))) after lookup
20:04:06.972 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:04:06.973 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] Successfully connected to peer
20:04:07.002 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0x5b845c22, L:/127.0.0.1:43632 - R:/127.6.32.193:39013] using TOKEN/null
20:04:07.068 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.32.195:39759
20:04:07.069 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:04:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:07.071866  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:04:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:04:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:04:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data
20:04:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/logs
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.195:39759
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.195
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36303
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.195
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:07.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:07.116045  6507 debug-util.cc:398] Leaking SignalData structure 0x7b08000a9280 after lost signal to thread 6438
20:04:07.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:07.117008  6507 debug-util.cc:398] Leaking SignalData structure 0x7b08000aec00 after lost signal to thread 6510
20:04:07.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:07.198820  7871 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 -> Peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Couldn't send request to peer 9e361d22f8464508ab46b5ee78fb2a3a. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:04:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:07.671779  8229 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:07.672475  8229 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:07.673413  8229 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:07.752651  8229 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:07.755023  8229 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.195
20:04:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:07.837291  8229 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.195:39759
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.195
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36303
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.195
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/logs
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:07.839486  8229 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:07.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:07.841857  8229 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:04:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:07.891678  8237 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:09.256433  7871 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111) [suppressed 10 similar messages]
20:04:09.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:09.632858  7871 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 -> Peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Couldn't send request to peer 9e361d22f8464508ab46b5ee78fb2a3a. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
20:04:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:09.282361  8236 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 8229
20:04:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:10.215570  8239 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2318 milliseconds
20:04:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:10.214746  8229 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.320s	user 0.000s	sys 0.002s
20:04:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:07.893142  8238 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:10.215994  8229 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.322s	user 0.000s	sys 0.002s
20:04:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.217353  8229 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:10.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:10.217418  8241 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:10.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.221529  8229 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.224643  8229 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.226289  8229 hybrid_clock.cc:648] HybridClock initialized: now 1783541050226238 us; error 63 us; skew 500 ppm
20:04:10.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.227245  8229 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.236719  8229 webserver.cc:533] Webserver started at http://127.6.32.195:36303/ using document root <none> and password file <none>
20:04:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.237982  8229 fs_manager.cc:362] Metadata directory not provided
20:04:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.238238  8229 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:10.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.248202  8229 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.008s	sys 0.000s
20:04:10.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.254480  8252 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:10.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.256067  8229 fs_manager.cc:730] Time spent opening block manager: real 0.006s	user 0.002s	sys 0.004s
20:04:10.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.256673  8229 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:04:10.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e361d22f8464508ab46b5ee78fb2a3a"
20:04:10.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:31 on dist-test-slave-xfg5"
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.260079  8229 fs_report.cc:389] FS layout report
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/data
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.339903  8229 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.342253  8229 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.343079  8229 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.346982  8229 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.353185  8259 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.363061  8229 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.363337  8229 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.012s	user 0.002s	sys 0.000s
20:04:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.363731  8229 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:10.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.369706  8229 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:10.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.370023  8229 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.004s	sys 0.000s
20:04:10.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.370514  8259 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap starting.
20:04:10.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.461792  8259 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Log is configured to *not* fsync() on all Append() calls
20:04:10.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.558519  8259 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.559520  8259 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap complete.
20:04:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.561363  8259 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Time spent bootstrapping tablet: real 0.191s	user 0.142s	sys 0.043s
20:04:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.588831  8259 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:10.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.592854  8259 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e361d22f8464508ab46b5ee78fb2a3a, State: Initialized, Role: FOLLOWER
20:04:10.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.594854  8259 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:10.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.599399  8229 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.195:39759
20:04:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.599767  8368 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.195:39759 every 8 connection(s)
20:04:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.601135  8259 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Time spent starting tablet: real 0.039s	user 0.039s	sys 0.000s
20:04:10.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.602777  8229 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:04:10.609 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.607496  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 8229
20:04:10.662 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:10.662 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.32.193:39013
20:04:10.663 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:04:10.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.664219  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 7848
20:04:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.675495  8371 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:04:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.676072  8371 heartbeater.cc:461] Registering TS with master...
20:04:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.677649  8371 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:04:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.678444  8369 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:04:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.678853  8369 heartbeater.cc:461] Registering TS with master...
20:04:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.679101  8370 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:04:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.679428  8370 heartbeater.cc:461] Registering TS with master...
20:04:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.679848  8369 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:04:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.680248  8370 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:04:10.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.682935  6321 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.683588  6396 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.686740  6474 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:10.689635  6474 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.195:50957
20:04:10.709 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] handling channelInactive
20:04:10.709 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state READY due to: connection closed
20:04:10.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:10.711 [DEBUG - Time-limited test] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:04:10.713 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.713 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.729 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:10.730 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:10.730 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1416954551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@6951e7aa -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72c6ddee -> wakeup thread Time-limited test)) after lookup
20:04:10.731 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:04:10.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.733 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.733 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.750 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:10.750 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:10.750 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [22ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [22ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [38ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1416954551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@6951e7aa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1755516220 after retry RPC@751735915), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72c6ddee -> wakeup thread Time-limited test -> (continuation of Deferred@1755516220 after retry RPC@751735915))) after lookup
20:04:10.751 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:04:10.752 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.752 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.752 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.788 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:10.789 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:10.789 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=78), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [22ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [22ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [38ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [40ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [41ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [41ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [77ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1416954551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@6951e7aa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72c6ddee -> wakeup thread Time-limited test -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483))) after lookup
20:04:10.790 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:04:10.791 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.791 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.791 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.869 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:10.869 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:10.871 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=160), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [22ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [22ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [38ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [40ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [41ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [41ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [77ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [78ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [80ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [80ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [158ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1416954551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@6951e7aa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483) -> (continuation of Deferred@734728762 after retry RPC@414578541), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72c6ddee -> wakeup thread Time-limited test -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483) -> (continuation of Deferred@734728762 after retry RPC@414578541))) after lookup
20:04:10.872 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:04:10.874 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.874 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:10.874 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:11.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:11.109 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:11.110 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=398), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [22ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [22ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [38ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [40ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [41ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [41ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [77ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [78ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [80ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [80ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [158ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [161ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [163ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [163ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [393ms] refreshing cache from master, [393ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [397ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1416954551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@6951e7aa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483) -> (continuation of Deferred@734728762 after retry RPC@414578541) -> (continuation of Deferred@960523390 after retry RPC@251263652), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72c6ddee -> wakeup thread Time-limited test -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483) -> (continuation of Deferred@734728762 after retry RPC@414578541) -> (continuation of Deferred@960523390 after retry RPC@251263652))) after lookup
20:04:11.111 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:04:11.112 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:11.112 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state CONNECTING due to: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:11.112 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013
20:04:11.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.693876  8370 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:04:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.878326  8381 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 1a9f83cc3ec74be2b5c075c8011c00f4)
20:04:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.878911  8381 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.881573  8381 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:11.885883  8057 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111)
20:04:11.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:11.891423  8057 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111)
20:04:11.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.896941  8322 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "8ba53d199a5c47b6acb057a49d2d0699" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" is_pre_election: true
20:04:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.897713  8322 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8ba53d199a5c47b6acb057a49d2d0699 in term 7.
20:04:11.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.899137  8057 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8ba53d199a5c47b6acb057a49d2d0699, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 1a9f83cc3ec74be2b5c075c8011c00f4
20:04:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.900007  8381 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 8 FOLLOWER]: Leader pre-election won for term 9
20:04:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.900321  8381 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 1a9f83cc3ec74be2b5c075c8011c00f4)
20:04:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.900664  8381 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 8 FOLLOWER]: Advancing to term 9
20:04:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.906116  8381 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.908151  8381 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 9 election: Requested vote from peers 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.910467  8322 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "8ba53d199a5c47b6acb057a49d2d0699" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "9e361d22f8464508ab46b5ee78fb2a3a"
20:04:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.911190  8322 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 7 FOLLOWER]: Advancing to term 9
20:04:11.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:11.913249  8057 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111)
20:04:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.919667  8322 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8ba53d199a5c47b6acb057a49d2d0699 in term 9.
20:04:11.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.920785  8057 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8ba53d199a5c47b6acb057a49d2d0699, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 1a9f83cc3ec74be2b5c075c8011c00f4
20:04:11.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.921497  8381 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 9 FOLLOWER]: Leader election won for term 9
20:04:11.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.923488  8381 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 9 LEADER]: Becoming Leader. State: Replica: 8ba53d199a5c47b6acb057a49d2d0699, State: Running, Role: LEADER
20:04:11.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.924783  8381 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:11.935885  6474 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 reported cstate change: term changed from 8 to 9, leader changed from 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193) to 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194). New cstate: current_term: 9 leader_uuid: "8ba53d199a5c47b6acb057a49d2d0699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: UNKNOWN } } }
20:04:12.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:12.430270  8057 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 -> Peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Couldn't send request to peer 1a9f83cc3ec74be2b5c075c8011c00f4. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:12.451241  8322 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 9 FOLLOWER]: Refusing update from remote peer 8ba53d199a5c47b6acb057a49d2d0699: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
20:04:12.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:12.453398  8381 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.001s
20:04:12.970 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:12.970 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:12.971 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2259), Traces: [0ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [2ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [18ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [19ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [22ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [22ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [38ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [40ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [41ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [41ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [73ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [77ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [78ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [80ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [80ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [158ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [161ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [163ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [163ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [393ms] refreshing cache from master, [393ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [397ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [399ms] sending RPC to server 1a9f83cc3ec74be2b5c075c8011c00f4, [401ms] received response from server 1a9f83cc3ec74be2b5c075c8011c00f4: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [401ms] delaying RPC due to: Network error: Failed to connect to peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013): Connection refused: /127.6.32.193:39013, [2253ms] refreshing cache from master, [2254ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [2259ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1416954551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@6951e7aa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483) -> (continuation of Deferred@734728762 after retry RPC@414578541) -> (continuation of Deferred@960523390 after retry RPC@251263652) -> (continuation of Deferred@59928027 after retry RPC@1124047831), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72c6ddee -> wakeup thread Time-limited test -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483) -> (continuation of Deferred@734728762 after retry RPC@414578541) -> (continuation of Deferred@960523390 after retry RPC@251263652) -> (continuation of Deferred@59928027 after retry RPC@1124047831))) after lookup
20:04:12.972 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:04:12.973 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] Successfully connected to peer
20:04:13.006 [DEBUG - kudu-nio-6] (Negotiator.java:918) Authenticated connection [id: 0x795a5864, L:/127.0.0.1:36164 - R:/127.6.32.194:41701] using TOKEN/null
20:04:13.069 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.32.193:39013
20:04:13.069 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:13.070986  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/logs
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.193:39013
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.193
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39967
20:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.193
20:04:13.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:04:13.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:13.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:13.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:13.451159  8417 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:13.451686  8417 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:13.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:13.452157  8417 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:13.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:13.491128  8417 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:13.492453  8417 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.193
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:13.536154  8417 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.193:39013
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:04:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.193
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39967
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.193
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/logs
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:13.537460  8417 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:13.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:13.539122  8417 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:04:13.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:13.555634  8425 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:14.980060  8057 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 -> Peer 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013): Couldn't send request to peer 1a9f83cc3ec74be2b5c075c8011c00f4. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.193:39013: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:04:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:14.959225  8424 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 8417
20:04:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:13.555990  8426 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:15.074982  8417 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.521s	user 0.460s	sys 1.012s
20:04:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:15.075641  8417 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.522s	user 0.461s	sys 1.012s
20:04:15.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:15.078258  8428 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:15.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:15.084247  8427 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1529 milliseconds
20:04:15.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.084247  8417 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.085740  8417 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:15.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.088156  8417 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:15.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.089603  8417 hybrid_clock.cc:648] HybridClock initialized: now 1783541055089552 us; error 60 us; skew 500 ppm
20:04:15.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.090816  8417 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:15.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.099620  8417 webserver.cc:533] Webserver started at http://127.6.32.193:39967/ using document root <none> and password file <none>
20:04:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.100919  8417 fs_manager.cc:362] Metadata directory not provided
20:04:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.101270  8417 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.112548  8417 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.006s	sys 0.002s
20:04:15.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.121667  8438 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:15.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.123958  8417 fs_manager.cc:730] Time spent opening block manager: real 0.008s	user 0.005s	sys 0.002s
20:04:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.124842  8417 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:04:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a9f83cc3ec74be2b5c075c8011c00f4"
20:04:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:26 on dist-test-slave-xfg5"
20:04:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.127712  8417 fs_report.cc:389] FS layout report
20:04:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:04:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/wal
20:04:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/data
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.196595  8417 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.198200  8417 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.198648  8417 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:15.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.201872  8417 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.209151  8445 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:04:15.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.221747  8417 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:15.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.222030  8417 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.015s	user 0.000s	sys 0.002s
20:04:15.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.222348  8417 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:15.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.227316  8417 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:15.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.227538  8417 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
20:04:15.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.227883  8445 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Bootstrap starting.
20:04:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.303457  8445 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Log is configured to *not* fsync() on all Append() calls
20:04:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.432186  8417 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.193:39013
20:04:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.432328  8552 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.193:39013 every 8 connection(s)
20:04:15.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.432889  8445 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:15.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.433707  8445 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Bootstrap complete.
20:04:15.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.435058  8445 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Time spent bootstrapping tablet: real 0.208s	user 0.145s	sys 0.056s
20:04:15.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.435694  8417 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-0/data/info.pb
20:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.444495  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 8417
20:04:15.445 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:15.485 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:15.486 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.32.194:41701
20:04:15.486 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:04:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.487267  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 8030
20:04:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.534404  8553 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:04:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.534981  8553 heartbeater.cc:461] Registering TS with master...
20:04:15.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.536382  8553 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:04:15.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.524516  8445 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:15.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.536790  8445 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a9f83cc3ec74be2b5c075c8011c00f4, State: Initialized, Role: FOLLOWER
20:04:15.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.537856  8445 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:15.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.545887  8554 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:04:15.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.546307  8554 heartbeater.cc:461] Registering TS with master...
20:04:15.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.547521  8554 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:04:15.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.549794  6395 ts_manager.cc:194] Re-registered known tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:04:15.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.550169  8557 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:04:15.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.550505  8557 heartbeater.cc:461] Registering TS with master...
20:04:15.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.551674  8557 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:04:15.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.552238  6475 ts_manager.cc:194] Re-registered known tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:04:15.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.555073  8445 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4: Time spent starting tablet: real 0.120s	user 0.034s	sys 0.000s
20:04:15.556 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:15.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.556594  6321 ts_manager.cc:194] Re-registered known tserver with Master: 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:04:15.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.558458  6475 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.193:54369
20:04:15.561 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] handling channelInactive
20:04:15.561 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state READY due to: connection closed
20:04:15.562 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: connection closed
20:04:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:15.564167  8554 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:04:15.570 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:15.571 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:15.571 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [5ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1110090284(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@7cb5a89d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b99d5d5 -> wakeup thread Time-limited test)) after lookup
20:04:15.572 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:04:15.574 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.574 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.574 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.588 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:15.588 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:15.588 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [5ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [15ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [30ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1110090284(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@7cb5a89d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2017393627 after retry RPC@1838984820), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b99d5d5 -> wakeup thread Time-limited test -> (continuation of Deferred@2017393627 after retry RPC@1838984820))) after lookup
20:04:15.589 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:04:15.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.590 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.628 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:15.629 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:15.629 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [5ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [15ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [30ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [32ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [33ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [33ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [71ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1110090284(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@7cb5a89d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2017393627 after retry RPC@1838984820) -> (continuation of Deferred@915979763 after retry RPC@628753144), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b99d5d5 -> wakeup thread Time-limited test -> (continuation of Deferred@2017393627 after retry RPC@1838984820) -> (continuation of Deferred@915979763 after retry RPC@628753144))) after lookup
20:04:15.629 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:04:15.630 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.630 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.630 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.750 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:15.750 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:15.750 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=193), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [5ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [15ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [30ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [32ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [33ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [33ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [71ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [72ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [73ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [73ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [188ms] refreshing cache from master, [188ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [192ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1110090284(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@7cb5a89d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2017393627 after retry RPC@1838984820) -> (continuation of Deferred@915979763 after retry RPC@628753144) -> (continuation of Deferred@1639402349 after retry RPC@338688097), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b99d5d5 -> wakeup thread Time-limited test -> (continuation of Deferred@2017393627 after retry RPC@1838984820) -> (continuation of Deferred@915979763 after retry RPC@628753144) -> (continuation of Deferred@1639402349 after retry RPC@338688097))) after lookup
20:04:15.752 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:04:15.753 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.753 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.753 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.990 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:15.990 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)[L],9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:15.992 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=435), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [5ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [15ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [30ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [32ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [33ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [33ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [71ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [72ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [73ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [73ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [188ms] refreshing cache from master, [188ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [192ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [194ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [196ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [196ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [427ms] refreshing cache from master, [428ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [432ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1110090284(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@7cb5a89d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2017393627 after retry RPC@1838984820) -> (continuation of Deferred@915979763 after retry RPC@628753144) -> (continuation of Deferred@1639402349 after retry RPC@338688097) -> (continuation of Deferred@621368947 after retry RPC@1317953091), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b99d5d5 -> wakeup thread Time-limited test -> (continuation of Deferred@2017393627 after retry RPC@1838984820) -> (continuation of Deferred@915979763 after retry RPC@628753144) -> (continuation of Deferred@1639402349 after retry RPC@338688097) -> (continuation of Deferred@621368947 after retry RPC@1317953091))) after lookup
20:04:15.993 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] connecting to peer
20:04:15.995 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.995 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:15.995 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701
20:04:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.904531  8567 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 8ba53d199a5c47b6acb057a49d2d0699)
20:04:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.905012  8567 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:16.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.907333  8567 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:04:16.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:16.912114  8255 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111)
20:04:16.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:16.916688  8255 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111)
20:04:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.920660  8508 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" is_pre_election: true
20:04:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.921324  8508 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e361d22f8464508ab46b5ee78fb2a3a in term 8.
20:04:16.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.922574  8256 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 8ba53d199a5c47b6acb057a49d2d0699
20:04:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.923609  8567 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 9 FOLLOWER]: Leader pre-election won for term 10
20:04:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.924409  8567 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 9 FOLLOWER]: Starting leader election (detected failure of leader 8ba53d199a5c47b6acb057a49d2d0699)
20:04:16.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.924841  8567 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 9 FOLLOWER]: Advancing to term 10
20:04:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.930445  8567 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:16.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.932591  8567 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 10 election: Requested vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193:39013)
20:04:16.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.933712  8508 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4"
20:04:16.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.934192  8508 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 8 FOLLOWER]: Advancing to term 10
20:04:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:16.937098  8255 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111)
20:04:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.941262  8508 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e361d22f8464508ab46b5ee78fb2a3a in term 10.
20:04:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.942257  8256 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 9e361d22f8464508ab46b5ee78fb2a3a; no voters: 8ba53d199a5c47b6acb057a49d2d0699
20:04:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.943138  8567 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 10 FOLLOWER]: Leader election won for term 10
20:04:16.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.945101  8567 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 10 LEADER]: Becoming Leader. State: Replica: 9e361d22f8464508ab46b5ee78fb2a3a, State: Running, Role: LEADER
20:04:16.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.946027  8567 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:16.956734  6475 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a reported cstate change: term changed from 9 to 10, leader changed from 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194) to 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195). New cstate: current_term: 10 leader_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: HEALTHY } } }
20:04:17.249 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:17.249 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:17.250 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1693), Traces: [0ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [5ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [15ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [17ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [30ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [32ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [33ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [33ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [71ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [72ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [73ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [73ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [188ms] refreshing cache from master, [188ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [192ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [194ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [196ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [196ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [427ms] refreshing cache from master, [428ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [432ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [436ms] sending RPC to server 8ba53d199a5c47b6acb057a49d2d0699, [438ms] received response from server 8ba53d199a5c47b6acb057a49d2d0699: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [438ms] delaying RPC due to: Network error: Failed to connect to peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701): Connection refused: /127.6.32.194:41701, [1687ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [1692ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1110090284(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@7cb5a89d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2017393627 after retry RPC@1838984820) -> (continuation of Deferred@915979763 after retry RPC@628753144) -> (continuation of Deferred@1639402349 after retry RPC@338688097) -> (continuation of Deferred@621368947 after retry RPC@1317953091) -> (continuation of Deferred@529262019 after retry RPC@592174135), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b99d5d5 -> wakeup thread Time-limited test -> (continuation of Deferred@2017393627 after retry RPC@1838984820) -> (continuation of Deferred@915979763 after retry RPC@628753144) -> (continuation of Deferred@1639402349 after retry RPC@338688097) -> (continuation of Deferred@621368947 after retry RPC@1317953091) -> (continuation of Deferred@529262019 after retry RPC@592174135))) after lookup
20:04:17.250 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:17.252 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] Successfully connected to peer
20:04:17.284 [DEBUG - kudu-nio-4] (Negotiator.java:918) Authenticated connection [id: 0xab463719, L:/127.0.0.1:45712 - R:/127.6.32.195:39759] using TOKEN/null
20:04:17.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:17.298277  8508 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 10 FOLLOWER]: Refusing update from remote peer 9e361d22f8464508ab46b5ee78fb2a3a: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 38. (index mismatch)
20:04:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:17.300254  8572 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.001s
20:04:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:17.301945  8255 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a -> Peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Couldn't send request to peer 8ba53d199a5c47b6acb057a49d2d0699. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:17.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:17.320555  8587 mvcc.cc:204] Tried to move back new op lower bound from 7305384153183465472 to 7305384148693463040. Current Snapshot: MvccSnapshot[applied={T|T < 7305384153183465472}]
20:04:17.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:17.332700  8585 mvcc.cc:204] Tried to move back new op lower bound from 7305384170663129088 to 7305384169262129152. Current Snapshot: MvccSnapshot[applied={T|T < 7305384170663129088}]
20:04:17.380 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.32.194:41701
20:04:17.381 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:17.382723  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/logs
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.194:41701
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.194
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33139
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.194
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:17.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:17.771941  8593 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:17.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:17.772537  8593 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:17.773043  8593 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:17.808786  8593 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:17.809666  8593 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.194
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:17.849009  8593 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.194:41701
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.194
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33139
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.194
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/logs
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:17.850482  8593 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:17.852134  8593 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:04:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:17.868769  8600 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:19.271766  8599 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 8593
20:04:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:19.554870  8599 kernel_stack_watchdog.cc:198] Thread 8593 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
20:04:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:17.872807  8601 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:19.556370  8593 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.685s	user 0.000s	sys 0.002s
20:04:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:19.556703  8593 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.685s	user 0.000s	sys 0.002s
20:04:19.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:19.560739  8603 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1682 milliseconds
20:04:19.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:19.562091  8606 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:19.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.562198  8593 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:19.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.563745  8593 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.566591  8593 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:19.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.568089  8593 hybrid_clock.cc:648] HybridClock initialized: now 1783541059568007 us; error 99 us; skew 500 ppm
20:04:19.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.568902  8593 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.575662  8593 webserver.cc:533] Webserver started at http://127.6.32.194:33139/ using document root <none> and password file <none>
20:04:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.576645  8593 fs_manager.cc:362] Metadata directory not provided
20:04:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.576861  8593 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.585188  8593 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.003s
20:04:19.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.590226  8615 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.591935  8593 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
20:04:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.592345  8593 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:04:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:04:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:28 on dist-test-slave-xfg5"
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.595271  8593 fs_report.cc:389] FS layout report
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/wal
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/data
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.663372  8593 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.664889  8593 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.665307  8593 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.667984  8593 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.674646  8622 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:04:19.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.683917  8593 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:19.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.684366  8593 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.001s
20:04:19.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.684619  8593 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:19.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.689899  8593 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:19.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.690114  8593 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.003s	sys 0.000s
20:04:19.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.690743  8622 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Bootstrap starting.
20:04:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.769475  8622 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Log is configured to *not* fsync() on all Append() calls
20:04:19.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:19.876788  8255 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a -> Peer 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701): Couldn't send request to peer 8ba53d199a5c47b6acb057a49d2d0699. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.194:41701: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:04:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.896788  8593 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.194:41701
20:04:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.897023  8733 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.194:41701 every 8 connection(s)
20:04:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.899801  8593 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-1/data/info.pb
20:04:19.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.902357  8622 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.903378  8622 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Bootstrap complete.
20:04:19.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.905519  8622 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Time spent bootstrapping tablet: real 0.215s	user 0.167s	sys 0.038s
20:04:19.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.908463  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 8593
20:04:19.909 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:19.940 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:19.941 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.32.195:39759
20:04:19.941 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:04:19.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.942894  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 8229
20:04:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.938472  8622 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.949141  8622 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ba53d199a5c47b6acb057a49d2d0699, State: Initialized, Role: FOLLOWER
20:04:19.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.954064  8622 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.968158  8734 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:04:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.968819  8734 heartbeater.cc:461] Registering TS with master...
20:04:19.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.970652  8734 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:04:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.977367  6396 ts_manager.cc:194] Re-registered known tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:04:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.978894  8622 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699: Time spent starting tablet: real 0.073s	user 0.047s	sys 0.008s
20:04:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.984717  8735 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:04:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.985129  8735 heartbeater.cc:461] Registering TS with master...
20:04:19.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.985942  8735 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:04:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.988827  8736 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:04:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.989224  8736 heartbeater.cc:461] Registering TS with master...
20:04:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.989332  6474 ts_manager.cc:194] Re-registered known tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:04:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.990199  8736 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:04:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.991611  6474 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.194:42783
20:04:19.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.994179  6321 ts_manager.cc:194] Re-registered known tserver with Master: 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701)
20:04:19.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:19.995754  8735 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:04:19.998 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] handling channelInactive
20:04:19.998 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state READY due to: connection closed
20:04:20.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:20.001 [DEBUG - Time-limited test] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:20.002 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.002 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.002 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.028 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:20.029 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:20.029 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [1ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [27ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1933502587(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@e69384f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ce1f9bb -> wakeup thread Time-limited test)) after lookup
20:04:20.029 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:20.030 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.030 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.030 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:20.048 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:20.049 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [1ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [27ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [28ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [29ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [29ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [47ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1933502587(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@e69384f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ce1f9bb -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912))) after lookup
20:04:20.049 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:20.051 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.051 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.051 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.089 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:20.089 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:20.089 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=88), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [1ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [27ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [28ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [29ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [29ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [47ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [48ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [50ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [50ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [87ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1933502587(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@e69384f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912) -> (continuation of Deferred@706928021 after retry RPC@1998181658), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ce1f9bb -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912) -> (continuation of Deferred@706928021 after retry RPC@1998181658))) after lookup
20:04:20.090 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:20.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.091 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.128 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:20.128 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:20.128 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=127), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [1ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [27ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [28ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [29ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [29ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [47ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [48ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [50ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [50ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [87ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [88ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [90ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [90ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [123ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [126ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1933502587(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@e69384f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912) -> (continuation of Deferred@706928021 after retry RPC@1998181658) -> (continuation of Deferred@1306559455 after retry RPC@786385595), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ce1f9bb -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912) -> (continuation of Deferred@706928021 after retry RPC@1998181658) -> (continuation of Deferred@1306559455 after retry RPC@786385595))) after lookup
20:04:20.129 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:20.130 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.130 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.130 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.309 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:20.310 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:20.310 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=309), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [1ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [27ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [28ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [29ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [29ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [47ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [48ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [50ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [50ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [87ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [88ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [90ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [90ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [123ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [126ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [128ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [129ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [130ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [304ms] refreshing cache from master, [304ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [308ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1933502587(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@e69384f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912) -> (continuation of Deferred@706928021 after retry RPC@1998181658) -> (continuation of Deferred@1306559455 after retry RPC@786385595) -> (continuation of Deferred@454220617 after retry RPC@1338818562), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ce1f9bb -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912) -> (continuation of Deferred@706928021 after retry RPC@1998181658) -> (continuation of Deferred@1306559455 after retry RPC@786385595) -> (continuation of Deferred@454220617 after retry RPC@1338818562))) after lookup
20:04:20.311 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:20.312 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.312 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.312 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.468 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:20.469 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013),8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)[L]]}]
20:04:20.469 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=468), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [1ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [27ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [28ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [29ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [29ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [47ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [48ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [50ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [50ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [87ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [88ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [90ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [90ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [123ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [126ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [128ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [129ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [130ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [304ms] refreshing cache from master, [304ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [308ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [310ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [311ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [311ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [463ms] refreshing cache from master, [464ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [467ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1933502587(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@e69384f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912) -> (continuation of Deferred@706928021 after retry RPC@1998181658) -> (continuation of Deferred@1306559455 after retry RPC@786385595) -> (continuation of Deferred@454220617 after retry RPC@1338818562) -> (continuation of Deferred@1801104068 after retry RPC@1119857533), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ce1f9bb -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912) -> (continuation of Deferred@706928021 after retry RPC@1998181658) -> (continuation of Deferred@1306559455 after retry RPC@786385595) -> (continuation of Deferred@454220617 after retry RPC@1338818562) -> (continuation of Deferred@1801104068 after retry RPC@1119857533))) after lookup
20:04:20.471 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] connecting to peer
20:04:20.472 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.472 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:20.472 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759) for tablet a41f0e13a5c84a95831681171defffc3: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759
20:04:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.373968  8748 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 9e361d22f8464508ab46b5ee78fb2a3a)
20:04:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.374472  8748 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.376914  8748 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:21.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:21.380396  8442 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:04:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:21.384627  8442 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:04:21.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.390316  8687 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699" is_pre_election: true
20:04:21.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.391211  8687 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a9f83cc3ec74be2b5c075c8011c00f4 in term 9.
20:04:21.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.392839  8441 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 8ba53d199a5c47b6acb057a49d2d0699; no voters: 9e361d22f8464508ab46b5ee78fb2a3a
20:04:21.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.393667  8748 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 10 FOLLOWER]: Leader pre-election won for term 11
20:04:21.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.394022  8748 raft_consensus.cc:493] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 9e361d22f8464508ab46b5ee78fb2a3a)
20:04:21.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.394425  8748 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 10 FOLLOWER]: Advancing to term 11
20:04:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.399317  8748 raft_consensus.cc:515] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.401384  8748 leader_election.cc:290] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 11 election: Requested vote from peers 8ba53d199a5c47b6acb057a49d2d0699 (127.6.32.194:41701), 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.402269  8687 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a41f0e13a5c84a95831681171defffc3" candidate_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "8ba53d199a5c47b6acb057a49d2d0699"
20:04:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.402917  8687 raft_consensus.cc:3060] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 9 FOLLOWER]: Advancing to term 11
20:04:21.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:21.406883  8442 leader_election.cc:336] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111)
20:04:21.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.411087  8687 raft_consensus.cc:2468] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a9f83cc3ec74be2b5c075c8011c00f4 in term 11.
20:04:21.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.412262  8441 leader_election.cc:304] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a9f83cc3ec74be2b5c075c8011c00f4, 8ba53d199a5c47b6acb057a49d2d0699; no voters: 9e361d22f8464508ab46b5ee78fb2a3a
20:04:21.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.412990  8748 raft_consensus.cc:2804] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 11 FOLLOWER]: Leader election won for term 11
20:04:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.414808  8748 raft_consensus.cc:697] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [term 11 LEADER]: Becoming Leader. State: Replica: 1a9f83cc3ec74be2b5c075c8011c00f4, State: Running, Role: LEADER
20:04:21.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.416493  8748 consensus_queue.cc:237] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.425274  6474 catalog_manager.cc:5697] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 reported cstate change: term changed from 10 to 11, leader changed from 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195) to 1a9f83cc3ec74be2b5c075c8011c00f4 (127.6.32.193). New cstate: current_term: 11 leader_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } health_report { overall_health: UNKNOWN } } }
20:04:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.943585  8687 raft_consensus.cc:1275] T a41f0e13a5c84a95831681171defffc3 P 8ba53d199a5c47b6acb057a49d2d0699 [term 11 FOLLOWER]: Refusing update from remote peer 1a9f83cc3ec74be2b5c075c8011c00f4: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
20:04:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:21.945058  8766 consensus_queue.cc:1048] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
20:04:21.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:21.989197  8442 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 -> Peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Couldn't send request to peer 9e361d22f8464508ab46b5ee78fb2a3a. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:24.289 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a41f0e13a5c84a95831681171defffc3 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789' with partition [<start>, <end>)
20:04:24.290 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a41f0e13a5c84a95831681171defffc3@[1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)[L],8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701),9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)]}]
20:04:24.290 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a41f0e13a5c84a95831681171defffc3, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=4289), Traces: [0ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [1ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [27ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [28ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [29ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [29ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [47ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [48ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [50ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [50ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [87ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [88ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [90ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [90ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [123ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [126ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [128ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [129ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [130ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [304ms] refreshing cache from master, [304ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [308ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [310ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [311ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [311ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [463ms] refreshing cache from master, [464ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [467ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, [469ms] sending RPC to server 9e361d22f8464508ab46b5ee78fb2a3a, [471ms] received response from server 9e361d22f8464508ab46b5ee78fb2a3a: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [471ms] delaying RPC due to: Network error: Failed to connect to peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759): Connection refused: /127.6.32.195:39759, [4283ms] refreshing cache from master, [4283ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.32.252:42851, [4288ms] Sub RPC GetTableLocations: received response from server master-127.6.32.252:42851: OK, deferred=Deferred@1933502587(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007ff030338f98@e69384f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912) -> (continuation of Deferred@706928021 after retry RPC@1998181658) -> (continuation of Deferred@1306559455 after retry RPC@786385595) -> (continuation of Deferred@454220617 after retry RPC@1338818562) -> (continuation of Deferred@1801104068 after retry RPC@1119857533) -> (continuation of Deferred@1152840423 after retry RPC@2070760783), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ce1f9bb -> wakeup thread Time-limited test -> (continuation of Deferred@1106308740 after retry RPC@583002912) -> (continuation of Deferred@706928021 after retry RPC@1998181658) -> (continuation of Deferred@1306559455 after retry RPC@786385595) -> (continuation of Deferred@454220617 after retry RPC@1338818562) -> (continuation of Deferred@1801104068 after retry RPC@1119857533) -> (continuation of Deferred@1152840423 after retry RPC@2070760783))) after lookup
20:04:24.290 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] connecting to peer
20:04:24.292 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] Successfully connected to peer
20:04:24.315 [DEBUG - kudu-nio-4] (Negotiator.java:918) Authenticated connection [id: 0x2241e936, L:/127.0.0.1:40310 - R:/127.6.32.193:39013] using TOKEN/null
20:04:24.374 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.32.195:39759
20:04:24.375 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:24.376215  6275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu
20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data
20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/logs
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.195:39759
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.195
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36303
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.195
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:24.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:24.488821  8442 consensus_peers.cc:597] T a41f0e13a5c84a95831681171defffc3 P 1a9f83cc3ec74be2b5c075c8011c00f4 -> Peer 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759): Couldn't send request to peer 9e361d22f8464508ab46b5ee78fb2a3a. Status: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:04:24.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:24.717231  8793 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:24.717923  8793 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:24.718454  8793 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:24.759786  8793 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:24.760680  8793 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.32.195
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:24.801644  8793 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.32.212:41205
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.32.195:39759
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.32.195
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36303
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.32.254:43103,127.6.32.253:39891,127.6.32.252:42851
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.32.195
20:04:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/logs
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:23 UTC on e7f111948823
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12779
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:24.803004  8793 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:24.804625  8793 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:04:24.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:24.818275  8801 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:26.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:26.410853  8793 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.593s	user 0.583s	sys 0.987s
20:04:26.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:26.222306  8800 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 8793
20:04:26.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:26.411317  8793 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.594s	user 0.583s	sys 0.987s
20:04:26.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:24.818419  8802 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:26.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:26.414381  8804 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:26.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:26.422564  8803 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1602 milliseconds
20:04:26.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.422588  8793 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:26.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.424468  8793 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:26.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.427487  8793 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.431193  8793 hybrid_clock.cc:648] HybridClock initialized: now 1783541066431017 us; error 166 us; skew 500 ppm
20:04:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.432456  8793 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.441478  8793 webserver.cc:533] Webserver started at http://127.6.32.195:36303/ using document root <none> and password file <none>
20:04:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.442835  8793 fs_manager.cc:362] Metadata directory not provided
20:04:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.443132  8793 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.455283  8793 fs_manager.cc:714] Time spent opening directory manager: real 0.008s	user 0.007s	sys 0.000s
20:04:26.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.460975  8817 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.462567  8793 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.006s	sys 0.000s
20:04:26.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.463025  8793 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data,/tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:04:26.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e361d22f8464508ab46b5ee78fb2a3a"
20:04:26.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 20:03:31 on dist-test-slave-xfg5"
20:04:26.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.465210  8793 fs_report.cc:389] FS layout report
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/wal
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/data
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:26.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.519761  8793 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:26.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.522035  8793 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.522747  8793 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 20:04:26.523898  8442 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.32.195:39759: connect: Connection refused (error 111) [suppressed 10 similar messages]
20:04:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.526201  8793 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.532233  8824 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:04:26.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.540709  8793 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:26.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.541054  8793 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.000s
20:04:26.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.541318  8793 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:26.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.546502  8793 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:26.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.546758  8793 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.002s
20:04:26.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.547325  8824 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap starting.
20:04:26.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.625787  8824 log.cc:826] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Log is configured to *not* fsync() on all Append() calls
20:04:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.743410  8793 rpc_server.cc:307] RPC server started. Bound to: 127.6.32.195:39759
20:04:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.743562  8933 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.32.195:39759 every 8 connection(s)
20:04:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.747277  8793 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskZLZTjO/test-tmp/mini-kudu-cluster16384799970194565394/ts-2/data/info.pb
20:04:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.758795  6275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu as pid 8793
20:04:26.760 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:26.811 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:26.833 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783541011789, tablet=a41f0e13a5c84a95831681171defffc3, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:26.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.838342  8936 heartbeater.cc:344] Connected to a master server at 127.6.32.254:43103
20:04:26.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.838910  8936 heartbeater.cc:461] Registering TS with master...
20:04:26.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.840833  8936 heartbeater.cc:507] Master 127.6.32.254:43103 requested a full tablet report, sending...
20:04:26.842 [DEBUG - main] (Connection.java:575) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] disconnecting while in state READY
20:04:26.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.842549  8935 heartbeater.cc:344] Connected to a master server at 127.6.32.252:42851
20:04:26.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.842986  8935 heartbeater.cc:461] Registering TS with master...
20:04:26.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.843041  8934 heartbeater.cc:344] Connected to a master server at 127.6.32.253:39891
20:04:26.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.843410  8934 heartbeater.cc:461] Registering TS with master...
20:04:26.846 [DEBUG - main] (Connection.java:575) [peer 8ba53d199a5c47b6acb057a49d2d0699(127.6.32.194:41701)] disconnecting while in state TERMINATED
20:04:26.846 [DEBUG - main] (Connection.java:575) [peer 9e361d22f8464508ab46b5ee78fb2a3a(127.6.32.195:39759)] disconnecting while in state TERMINATED
20:04:26.846 [DEBUG - main] (Connection.java:575) [peer master-127.6.32.253:39891(127.6.32.253:39891)] disconnecting while in state READY
20:04:26.850 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] handling channelInactive
20:04:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.844067  8935 heartbeater.cc:507] Master 127.6.32.252:42851 requested a full tablet report, sending...
20:04:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.844300  8934 heartbeater.cc:507] Master 127.6.32.253:39891 requested a full tablet report, sending...
20:04:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.846998  6320 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:26.850 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 1a9f83cc3ec74be2b5c075c8011c00f4(127.6.32.193:39013)] cleaning up while in state READY due to: connection closed
20:04:26.850 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.6.32.253:39891(127.6.32.253:39891)] handling channelInactive
20:04:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.847736  6475 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:26.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.850919  6475 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.32.195:47137
20:04:26.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.852100  6396 ts_manager.cc:194] Re-registered known tserver with Master: 9e361d22f8464508ab46b5ee78fb2a3a (127.6.32.195:39759)
20:04:26.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.32.253:39891(127.6.32.253:39891)] cleaning up while in state READY due to: connection closed
20:04:26.853 [DEBUG - main] (Connection.java:575) [peer master-127.6.32.252:42851(127.6.32.252:42851)] disconnecting while in state READY
20:04:26.854 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.6.32.252:42851(127.6.32.252:42851)] handling channelInactive
20:04:26.855 [DEBUG - main] (Connection.java:575) [peer master-127.6.32.254:43103(127.6.32.254:43103)] disconnecting while in state READY
20:04:26.856 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.32.252:42851(127.6.32.252:42851)] cleaning up while in state READY due to: connection closed
20:04:26.857 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:04:26.859 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.6.32.254:43103(127.6.32.254:43103)] handling channelInactive
20:04:26.859 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.6.32.254:43103(127.6.32.254:43103)] cleaning up while in state READY due to: connection closed
20:04:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.859804  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 8417
20:04:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.863855  8824 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:26.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.864902  8824 tablet_bootstrap.cc:492] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Bootstrap complete.
20:04:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.866830  8824 ts_tablet_manager.cc:1403] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Time spent bootstrapping tablet: real 0.320s	user 0.220s	sys 0.050s
20:04:26.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.887669  8824 raft_consensus.cc:359] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:26.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.891297  8824 raft_consensus.cc:740] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e361d22f8464508ab46b5ee78fb2a3a, State: Initialized, Role: FOLLOWER
20:04:26.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.892333  8824 consensus_queue.cc:260] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ba53d199a5c47b6acb057a49d2d0699" member_type: VOTER last_known_addr { host: "127.6.32.194" port: 41701 } } peers { permanent_uuid: "1a9f83cc3ec74be2b5c075c8011c00f4" member_type: VOTER last_known_addr { host: "127.6.32.193" port: 39013 } } peers { permanent_uuid: "9e361d22f8464508ab46b5ee78fb2a3a" member_type: VOTER last_known_addr { host: "127.6.32.195" port: 39759 } }
20:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.896633  8824 ts_tablet_manager.cc:1434] T a41f0e13a5c84a95831681171defffc3 P 9e361d22f8464508ab46b5ee78fb2a3a: Time spent starting tablet: real 0.029s	user 0.025s	sys 0.003s
20:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.897565  8935 heartbeater.cc:499] Master 127.6.32.252:42851 was elected leader, sending a full tablet report...
20:04:26.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.912470  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 8593
20:04:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.938555  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 8793
20:04:26.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:26.968626  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 6287
20:04:27.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:27.002763  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 6358
20:04:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 20:04:27.035408  6275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskZLZTjO/build/tsan/bin/kudu with pid 6433
20:04:28.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T20:04:28Z chronyd exiting

Time: 74.564

OK (1 test)

