JUnit version 4.13.2
.12:12:00.420 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783512720411
12:12:00.487 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
12:12:00.528 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
12:12:00.541 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task8wlbrb/test-tmp
12:12:00.564 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
12:12:00.567 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task8wlbrb/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
12:12:00.772 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

12:12:00.946 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
12:12:00.947 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

12:12:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:00.952183 29982 mini_oidc.cc:150] Starting JWKS server
12:12:00.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:00.956203 29982 webserver.cc:533] Webserver started at http://127.0.0.1:36017/ using document root <none> and password file <none>
12:12:00.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:00.956795 29982 mini_oidc.cc:186] Starting OIDC Discovery server
12:12:00.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:00.959044 29982 webserver.cc:533] Webserver started at http://127.0.0.1:37751/ using document root <none> and password file <none>
12:12:01.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:12:01Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
12:12:01.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:12:01Z Disabled control of system clock
12:12:01.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:01.016871 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:01.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:01.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/wal
12:12:01.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/data
12:12:01.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:01.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:01.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:01.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/logs
12:12:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/data/info.pb
12:12:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:12:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:01.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:01.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:01.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.190:46473
12:12:01.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.190
12:12:01.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:01.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:01.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:01.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:01.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:01.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:01.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:01.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:01.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36017/jwks.json with env {}
12:12:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.304755 29995 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.305325 29995 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.305765 29995 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:01.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.337026 29995 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:12:01.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.337311 29995 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:01.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.337502 29995 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:12:01.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.337695 29995 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:12:01.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.337886 29995 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:12:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.338024 29995 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36017/jwks.json
12:12:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:01.373963 29995 master_runner.cc:387] Master server non-default flags:
12:12:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/data
12:12:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/wal
12:12:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.190:46473
12:12:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36017/jwks.json
12:12:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:01.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:01.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/data/info.pb
12:12:01.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.190
12:12:01.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:01.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:01.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:01.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:01.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:01.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:01.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/logs
12:12:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:12:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:01.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:01.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:01.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:01.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:01.375281 29995 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:01.376843 29995 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:01.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.387072 30001 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:02.790084 30000 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 29995
12:12:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:02.923502 29995 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.536s	user 0.637s	sys 0.899s
12:12:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:02.923825 29995 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.537s	user 0.637s	sys 0.899s
12:12:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:01.387739 30002 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:02.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:02.927372 30003 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1540 milliseconds
12:12:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:02.928647 30004 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.928745 29995 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.930168 29995 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:02.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.932626 29995 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:02.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.933985 29995 hybrid_clock.cc:648] HybridClock initialized: now 1783512722933943 us; error 35 us; skew 500 ppm
12:12:02.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.934891 29995 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.941038 29995 webserver.cc:533] Webserver started at http://127.29.71.190:43239/ using document root <none> and password file <none>
12:12:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.942025 29995 fs_manager.cc:362] Metadata directory not provided
12:12:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.942241 29995 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:02.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.942857 29995 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:02.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.947124 29995 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/data/instance:
12:12:02.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a15ac30a6f74491595e213efe816c97d"
12:12:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:02 on dist-test-slave-tp5p"
12:12:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.948582 29995 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/wal/instance:
12:12:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a15ac30a6f74491595e213efe816c97d"
12:12:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:02 on dist-test-slave-tp5p"
12:12:02.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.955518 29995 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.000s
12:12:02.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.961211 30011 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:02.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.962220 29995 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.003s
12:12:02.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.962560 29995 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/wal
12:12:02.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a15ac30a6f74491595e213efe816c97d"
12:12:02.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:02 on dist-test-slave-tp5p"
12:12:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:02.962982 29995 fs_report.cc:389] FS layout report
12:12:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/wal
12:12:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/wal
12:12:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/data/data
12:12:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:02.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:02.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:03.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.031675 29995 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:03.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.033386 29995 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:03.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.033918 29995 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:03.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.103657 29995 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.190:46473
12:12:03.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.103756 30062 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.190:46473 every 8 connection(s)
12:12:03.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.106556 29995 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/data/info.pb
12:12:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.112299 30063 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.115294 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 29995
12:12:03.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.115710 29982 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-0/wal/instance
12:12:03.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.118968 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:03.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:03.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/wal
12:12:03.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/data
12:12:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/logs
12:12:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/data/info.pb
12:12:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:03.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:03.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:03.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:12:03.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:03.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:03.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.189:40761
12:12:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.189
12:12:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36017/jwks.json with env {}
12:12:03.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.122848 30063 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } has no permanent_uuid. Determining permanent_uuid...
12:12:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.154874 30063 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } has no permanent_uuid. Determining permanent_uuid...
12:12:03.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.157254 30012 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.189:40761: connect: Connection refused (error 111)
12:12:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.161432 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.189:40761: Network error: Client connection negotiation failed: client connection to 127.29.71.189:40761: connect: Connection refused (error 111)
12:12:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.210862 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } attempt: 1
12:12:03.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.215521 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.189:40761: Network error: Client connection negotiation failed: client connection to 127.29.71.189:40761: connect: Connection refused (error 111)
12:12:03.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.283891 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } attempt: 2
12:12:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.288028 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.189:40761: Network error: Client connection negotiation failed: client connection to 127.29.71.189:40761: connect: Connection refused (error 111)
12:12:03.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.379494 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } attempt: 3
12:12:03.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.384009 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.189:40761: Network error: Client connection negotiation failed: client connection to 127.29.71.189:40761: connect: Connection refused (error 111)
12:12:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.414825 30065 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.415495 30065 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.416242 30065 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:03.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.447463 30065 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:12:03.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.447799 30065 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:03.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.448109 30065 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:12:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.448459 30065 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:12:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.448804 30065 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:12:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.449113 30065 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36017/jwks.json
12:12:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.488412 30065 master_runner.cc:387] Master server non-default flags:
12:12:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/data
12:12:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/wal
12:12:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.189:40761
12:12:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36017/jwks.json
12:12:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/data/info.pb
12:12:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.189
12:12:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/logs
12:12:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:03.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:03.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:12:03.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:03.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:03.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:03.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:03.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.490082 30065 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.492276 30065 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.505302 30074 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:03.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.527381 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } attempt: 4
12:12:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.532776 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.189:40761: Network error: Client connection negotiation failed: client connection to 127.29.71.189:40761: connect: Connection refused (error 111)
12:12:03.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:03.832249 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } attempt: 5
12:12:03.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.839097 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.189:40761: Network error: Client connection negotiation failed: client connection to 127.29.71.189:40761: connect: Connection refused (error 111)
12:12:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.386571 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } attempt: 6
12:12:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:04.395268 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.189:40761: Network error: Client connection negotiation failed: client connection to 127.29.71.189:40761: connect: Connection refused (error 111)
12:12:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.505870 30075 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:04.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:03.508805 30077 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:04.618029 30076 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1110 milliseconds
12:12:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.618186 30065 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:04.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.619592 30065 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.622189 30065 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.623544 30065 hybrid_clock.cc:648] HybridClock initialized: now 1783512724623504 us; error 45 us; skew 500 ppm
12:12:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.624469 30065 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:04.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.630769 30065 webserver.cc:533] Webserver started at http://127.29.71.189:35303/ using document root <none> and password file <none>
12:12:04.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.631698 30065 fs_manager.cc:362] Metadata directory not provided
12:12:04.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.631904 30065 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:04.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.632458 30065 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:04.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.636596 30065 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/data/instance:
12:12:04.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "819562e8af58403dace7f792f6cfa1ed"
12:12:04.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:04 on dist-test-slave-tp5p"
12:12:04.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.637903 30065 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/wal/instance:
12:12:04.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "819562e8af58403dace7f792f6cfa1ed"
12:12:04.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:04 on dist-test-slave-tp5p"
12:12:04.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.644927 30065 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.000s
12:12:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.650416 30086 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.651443 30065 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.005s	sys 0.000s
12:12:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.651718 30065 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/wal
12:12:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "819562e8af58403dace7f792f6cfa1ed"
12:12:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:04 on dist-test-slave-tp5p"
12:12:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.652006 30065 fs_report.cc:389] FS layout report
12:12:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/wal
12:12:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/wal
12:12:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/data/data
12:12:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:04.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.720789 30065 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:04.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.722221 30065 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:04.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.722607 30065 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:04.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.788311 30065 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.189:40761
12:12:04.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.788429 30137 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.189:40761 every 8 connection(s)
12:12:04.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.791858 30065 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/data/info.pb
12:12:04.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.798507 30138 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.798893 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 30065
12:12:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.799350 29982 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-1/wal/instance
12:12:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.802353 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/wal
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/data
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/logs
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/data/info.pb
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:12:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.188:37701
12:12:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.188
12:12:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:04.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:04.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:04.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36017/jwks.json with env {}
12:12:04.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.811529 30138 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } has no permanent_uuid. Determining permanent_uuid...
12:12:04.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.832593 30138 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } has no permanent_uuid. Determining permanent_uuid...
12:12:04.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.845489 30138 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } has no permanent_uuid. Determining permanent_uuid...
12:12:04.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:04.847754 30089 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:04.851637 30138 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:04.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.901158 30138 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 1
12:12:04.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:04.905364 30138 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:04.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:04.973799 30138 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 2
12:12:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:04.977946 30138 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.069377 30138 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 3
12:12:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.073415 30138 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:05.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.095799 30140 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:05.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.096338 30140 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:05.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.096801 30140 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:05.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.131807 30140 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:12:05.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.132079 30140 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:05.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.132283 30140 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:12:05.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.132468 30140 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:12:05.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.132644 30140 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:12:05.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.132790 30140 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36017/jwks.json
12:12:05.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.168005 30140 master_runner.cc:387] Master server non-default flags:
12:12:05.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:05.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:05.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/data
12:12:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/wal
12:12:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.188:37701
12:12:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36017/jwks.json
12:12:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/data/info.pb
12:12:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.188
12:12:05.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:05.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:05.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:05.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:05.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:05.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:05.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/logs
12:12:05.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:12:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.169348 30140 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.170892 30140 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.181324 30150 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:05.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.217338 30138 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 4
12:12:05.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.224133 30138 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.455793 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } attempt: 7
12:12:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.487732 30063 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } has no permanent_uuid. Determining permanent_uuid...
12:12:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.504259 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:05.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.523586 30138 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 5
12:12:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.530586 30138 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.562737 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 1
12:12:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.570313 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.651782 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 2
12:12:05.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.658917 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:05.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.744374 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 3
12:12:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.751411 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:05.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:05.891902 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 4
12:12:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.899292 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.078089 30138 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 6
12:12:06.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.112609 30138 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.182765 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 5
12:12:06.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.189596 30063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.71.188:37701: Network error: Client connection negotiation failed: client connection to 127.29.71.188:37701: connect: Connection refused (error 111)
12:12:06.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:05.181872 30151 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.559746 30152 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1376 milliseconds
12:12:06.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.561424 30153 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:06.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.564167 30140 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.383s	user 0.489s	sys 0.879s
12:12:06.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.564473 30140 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.383s	user 0.489s	sys 0.879s
12:12:06.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.564751 30140 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:06.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.566012 30140 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:06.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.568713 30140 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:06.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.570106 30140 hybrid_clock.cc:648] HybridClock initialized: now 1783512726570058 us; error 59 us; skew 500 ppm
12:12:06.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.571007 30140 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:06.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.577919 30140 webserver.cc:533] Webserver started at http://127.29.71.188:37145/ using document root <none> and password file <none>
12:12:06.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.578851 30140 fs_manager.cc:362] Metadata directory not provided
12:12:06.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.579058 30140 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:06.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.579572 30140 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:06.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.583873 30140 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/data/instance:
12:12:06.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "043e0f9a2d9844a7996b199733328e8b"
12:12:06.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:06 on dist-test-slave-tp5p"
12:12:06.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.585152 30140 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/wal/instance:
12:12:06.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "043e0f9a2d9844a7996b199733328e8b"
12:12:06.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:06 on dist-test-slave-tp5p"
12:12:06.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.594466 30140 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.004s	sys 0.004s
12:12:06.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.600232 30165 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:06.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.601274 30140 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
12:12:06.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.601538 30140 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/wal
12:12:06.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "043e0f9a2d9844a7996b199733328e8b"
12:12:06.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:06 on dist-test-slave-tp5p"
12:12:06.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.601823 30140 fs_report.cc:389] FS layout report
12:12:06.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:06.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/wal
12:12:06.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/wal
12:12:06.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/data/data
12:12:06.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:06.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:06.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:06.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.665604 30140 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:06.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.667089 30140 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:06.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.667476 30140 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:06.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.733409 30140 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.188:37701
12:12:06.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.733480 30216 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.188:37701 every 8 connection(s)
12:12:06.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.736460 30140 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/data/info.pb
12:12:06.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.741809 30217 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:06.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.742054 30063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 6
12:12:06.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.746372 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 30140
12:12:06.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.746811 29982 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/master-2/wal/instance
12:12:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.762033 30217 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } has no permanent_uuid. Determining permanent_uuid...
12:12:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.779227 30063 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d: Bootstrap starting.
12:12:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.785194 30217 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } has no permanent_uuid. Determining permanent_uuid...
12:12:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.786679 30063 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d: Neither blocks nor log segments found. Creating new log.
12:12:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.789343 30063 log.cc:826] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d: Log is configured to *not* fsync() on all Append() calls
12:12:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.797670 30217 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } has no permanent_uuid. Determining permanent_uuid...
12:12:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.797940 30063 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d: No bootstrap required, opened a new log
12:12:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.819335 30217 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b: Bootstrap starting.
12:12:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.819942 30063 raft_consensus.cc:359] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } }
12:12:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.820578 30063 raft_consensus.cc:385] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:06.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.820863 30063 raft_consensus.cc:740] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a15ac30a6f74491595e213efe816c97d, State: Initialized, Role: FOLLOWER
12:12:06.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.821797 30063 consensus_queue.cc:260] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } }
12:12:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.828570 30217 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b: Neither blocks nor log segments found. Creating new log.
12:12:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.828135 30228 sys_catalog.cc:455] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } } }
12:12:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.828940 30228 sys_catalog.cc:458] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [sys.catalog]: This master's current role is: FOLLOWER
12:12:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.829988 30063 sys_catalog.cc:565] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [sys.catalog]: configured and running, proceeding with master startup.
12:12:06.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.831317 30217 log.cc:826] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b: Log is configured to *not* fsync() on all Append() calls
12:12:06.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.840850 30217 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b: No bootstrap required, opened a new log
12:12:06.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.868105 30240 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
12:12:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.868113 30217 raft_consensus.cc:359] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } }
12:12:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.868965 30217 raft_consensus.cc:385] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.869259 30217 raft_consensus.cc:740] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 043e0f9a2d9844a7996b199733328e8b, State: Initialized, Role: FOLLOWER
12:12:06.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.870253 30240 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
12:12:06.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.870479 30217 consensus_queue.cc:260] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } }
12:12:06.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.881916 30242 sys_catalog.cc:455] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } } }
12:12:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.883126 30242 sys_catalog.cc:458] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [sys.catalog]: This master's current role is: FOLLOWER
12:12:06.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.883858 30217 sys_catalog.cc:565] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [sys.catalog]: configured and running, proceeding with master startup.
12:12:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.863579 29995 master_runner.cc:187] Error getting master registration for 127.29.71.189:40761: OK, instance_id { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" instance_seqno: 1783512724770474 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
12:12:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.896600 29995 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
12:12:06.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.920069 30253 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
12:12:06.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.921749 30253 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
12:12:06.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:06.922348 30171 tablet.cc:2406] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:12:06.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.930250 30228 raft_consensus.cc:493] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:12:06.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.930693 30228 raft_consensus.cc:515] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } }
12:12:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.932900 30228 leader_election.cc:290] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 819562e8af58403dace7f792f6cfa1ed (127.29.71.189:40761), 043e0f9a2d9844a7996b199733328e8b (127.29.71.188:37701)
12:12:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.934419 30192 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a15ac30a6f74491595e213efe816c97d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "043e0f9a2d9844a7996b199733328e8b" is_pre_election: true
12:12:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.935259 30192 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a15ac30a6f74491595e213efe816c97d in term 0.
12:12:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.936611 30014 leader_election.cc:304] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 043e0f9a2d9844a7996b199733328e8b, a15ac30a6f74491595e213efe816c97d; no voters: 
12:12:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.937491 30228 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 0 FOLLOWER]: Leader pre-election won for term 1
12:12:06.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.937816 30228 raft_consensus.cc:493] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
12:12:06.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.938133 30228 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 0 FOLLOWER]: Advancing to term 1
12:12:06.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.944283 30228 raft_consensus.cc:515] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } }
12:12:06.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.952122 30192 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a15ac30a6f74491595e213efe816c97d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "043e0f9a2d9844a7996b199733328e8b"
12:12:06.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.952651 30192 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [term 0 FOLLOWER]: Advancing to term 1
12:12:06.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.922515 30140 master_runner.cc:187] Error getting master registration for 127.29.71.189:40761: OK, instance_id { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" instance_seqno: 1783512724770474 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
12:12:06.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.954429 30140 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
12:12:06.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.955996 30228 leader_election.cc:290] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [CANDIDATE]: Term 1 election: Requested vote from peers 819562e8af58403dace7f792f6cfa1ed (127.29.71.189:40761), 043e0f9a2d9844a7996b199733328e8b (127.29.71.188:37701)
12:12:06.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.958988 30192 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a15ac30a6f74491595e213efe816c97d in term 1.
12:12:06.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.963757 30014 leader_election.cc:304] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 043e0f9a2d9844a7996b199733328e8b, a15ac30a6f74491595e213efe816c97d; no voters: 
12:12:06.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.964785 30228 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 1 FOLLOWER]: Leader election won for term 1
12:12:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.968376 30228 raft_consensus.cc:697] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [term 1 LEADER]: Becoming Leader. State: Replica: a15ac30a6f74491595e213efe816c97d, State: Running, Role: LEADER
12:12:06.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.969615 30228 consensus_queue.cc:237] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } }
12:12:06.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.985773 30255 sys_catalog.cc:455] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [sys.catalog]: SysCatalogTable state changed. Reason: New leader a15ac30a6f74491595e213efe816c97d. Latest consensus state: current_term: 1 leader_uuid: "a15ac30a6f74491595e213efe816c97d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } } }
12:12:06.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.986572 30255 sys_catalog.cc:458] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [sys.catalog]: This master's current role is: LEADER
12:12:06.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.989909 30257 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
12:12:06.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:06.996981 30257 catalog_manager.cc:1520] Initializing Kudu cluster ID...
12:12:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.017642 30192 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [term 1 FOLLOWER]: Refusing update from remote peer a15ac30a6f74491595e213efe816c97d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:12:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.019537 30255 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [LEADER]: Connected to new peer: Peer: permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:12:07.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:07.043641 30017 tablet.cc:2406] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:12:07.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.043879 30242 sys_catalog.cc:455] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [sys.catalog]: SysCatalogTable state changed. Reason: New leader a15ac30a6f74491595e213efe816c97d. Latest consensus state: current_term: 1 leader_uuid: "a15ac30a6f74491595e213efe816c97d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } } }
12:12:07.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.044526 30242 sys_catalog.cc:458] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [sys.catalog]: This master's current role is: FOLLOWER
12:12:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.052119 30258 mvcc.cc:204] Tried to move back new op lower bound from 7305268129847357440 to 7305268129677570048. Current Snapshot: MvccSnapshot[applied={T|T < 7305268129847357440}]
12:12:07.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.057289 30259 mvcc.cc:204] Tried to move back new op lower bound from 7305268129847357440 to 7305268129677570048. Current Snapshot: MvccSnapshot[applied={T|T < 7305268129847357440}]
12:12:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.070896 30242 sys_catalog.cc:455] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a15ac30a6f74491595e213efe816c97d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } } }
12:12:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.071597 30242 sys_catalog.cc:458] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b [sys.catalog]: This master's current role is: FOLLOWER
12:12:07.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.074697 30255 sys_catalog.cc:455] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a15ac30a6f74491595e213efe816c97d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } } }
12:12:07.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.075389 30255 sys_catalog.cc:458] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [sys.catalog]: This master's current role is: LEADER
12:12:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.081053 30257 catalog_manager.cc:1383] Generated new cluster ID: 4067ad899bb44365a8192fe9635e10e3
12:12:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.081367 30257 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
12:12:07.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.125072 30257 catalog_manager.cc:1406] Generated new certificate authority record
12:12:07.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.130652 30257 catalog_manager.cc:1540] Loading token signing keys...
12:12:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.173098 30138 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } attempt: 7
12:12:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.174278 30257 catalog_manager.cc:6070] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d: Generated new TSK 0
12:12:07.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.176223 30257 catalog_manager.cc:1550] Initializing in-progress tserver states...
12:12:07.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.202107 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:07.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/logs
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.129:0
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.129
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.129
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.231421 30138 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed: Bootstrap starting.
12:12:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.250438 30138 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed: Neither blocks nor log segments found. Creating new log.
12:12:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.256086 30138 log.cc:826] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed: Log is configured to *not* fsync() on all Append() calls
12:12:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.277997 30138 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed: No bootstrap required, opened a new log
12:12:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.305774 30138 raft_consensus.cc:359] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } }
12:12:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.306553 30138 raft_consensus.cc:385] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:07.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.307833 30138 raft_consensus.cc:740] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 819562e8af58403dace7f792f6cfa1ed, State: Initialized, Role: FOLLOWER
12:12:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.308755 30138 consensus_queue.cc:260] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } }
12:12:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.329838 30266 sys_catalog.cc:455] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } } }
12:12:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.330603 30266 sys_catalog.cc:458] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [sys.catalog]: This master's current role is: FOLLOWER
12:12:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.331017 30266 raft_consensus.cc:493] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:12:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.331459 30266 raft_consensus.cc:515] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } }
12:12:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.331794 30138 sys_catalog.cc:565] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [sys.catalog]: configured and running, proceeding with master startup.
12:12:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.332414 30112 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a15ac30a6f74491595e213efe816c97d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "819562e8af58403dace7f792f6cfa1ed"
12:12:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.333797 30266 leader_election.cc:290] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a15ac30a6f74491595e213efe816c97d (127.29.71.190:46473), 043e0f9a2d9844a7996b199733328e8b (127.29.71.188:37701)
12:12:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.335876 30192 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "819562e8af58403dace7f792f6cfa1ed" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "043e0f9a2d9844a7996b199733328e8b" is_pre_election: true
12:12:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.337080 30113 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a15ac30a6f74491595e213efe816c97d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "819562e8af58403dace7f792f6cfa1ed" is_pre_election: true
12:12:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.339639 30111 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [term 0 FOLLOWER]: Advancing to term 1
12:12:07.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.350598 30111 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [term 1 FOLLOWER]: Refusing update from remote peer a15ac30a6f74491595e213efe816c97d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:12:07.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.367034 30038 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "819562e8af58403dace7f792f6cfa1ed" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a15ac30a6f74491595e213efe816c97d" is_pre_election: true
12:12:07.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.369913 30087 leader_election.cc:304] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 819562e8af58403dace7f792f6cfa1ed; no voters: 043e0f9a2d9844a7996b199733328e8b, a15ac30a6f74491595e213efe816c97d
12:12:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.371106 30266 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
12:12:07.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.374765 30255 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [LEADER]: Connected to new peer: Peer: permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:12:07.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.415342 30274 mvcc.cc:204] Tried to move back new op lower bound from 7305268129847357440 to 7305268129677570048. Current Snapshot: MvccSnapshot[applied={T|T < 7305268129847357440}]
12:12:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.439097 30266 sys_catalog.cc:455] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [sys.catalog]: SysCatalogTable state changed. Reason: New leader a15ac30a6f74491595e213efe816c97d. Latest consensus state: current_term: 1 leader_uuid: "a15ac30a6f74491595e213efe816c97d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } } }
12:12:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.440488 30266 sys_catalog.cc:458] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [sys.catalog]: This master's current role is: FOLLOWER
12:12:07.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.439189 30267 sys_catalog.cc:455] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a15ac30a6f74491595e213efe816c97d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } } }
12:12:07.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.451210 30267 sys_catalog.cc:458] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed [sys.catalog]: This master's current role is: FOLLOWER
12:12:07.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.460461 30283 catalog_manager.cc:1295] Loaded cluster ID: 4067ad899bb44365a8192fe9635e10e3
12:12:07.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.460708 30283 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed: loading cluster ID for follower catalog manager: success
12:12:07.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.467190 30283 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed: acquiring CA information for follower catalog manager: success
12:12:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.478838 30283 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 819562e8af58403dace7f792f6cfa1ed: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
12:12:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:07.856354 30264 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:07.857489 30264 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:07.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:07.858489 30264 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:07.917526 30264 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:07.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:07.919301 30264 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.129
12:12:07.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.927383 30253 catalog_manager.cc:1295] Loaded cluster ID: 4067ad899bb44365a8192fe9635e10e3
12:12:07.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.927635 30253 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b: loading cluster ID for follower catalog manager: success
12:12:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.940759 30253 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b: acquiring CA information for follower catalog manager: success
12:12:07.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.965888 30253 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 043e0f9a2d9844a7996b199733328e8b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
12:12:07.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.966691 30255 sys_catalog.cc:455] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a15ac30a6f74491595e213efe816c97d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a15ac30a6f74491595e213efe816c97d" member_type: VOTER last_known_addr { host: "127.29.71.190" port: 46473 } } peers { permanent_uuid: "819562e8af58403dace7f792f6cfa1ed" member_type: VOTER last_known_addr { host: "127.29.71.189" port: 40761 } } peers { permanent_uuid: "043e0f9a2d9844a7996b199733328e8b" member_type: VOTER last_known_addr { host: "127.29.71.188" port: 37701 } } }
12:12:07.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:07.967358 30255 sys_catalog.cc:458] T 00000000000000000000000000000000 P a15ac30a6f74491595e213efe816c97d [sys.catalog]: This master's current role is: LEADER
12:12:08.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:08.020197 30264 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.129:0
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.129
12:12:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.129
12:12:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/logs
12:12:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:08.031046 30264 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:08.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:08.033689 30264 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:08.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:08.062486 30293 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:08.067862 30294 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:09.455873 30292 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 30264
12:12:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:09.781922 30264 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.716s	user 0.454s	sys 0.596s
12:12:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:09.784458 30264 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.718s	user 0.454s	sys 0.596s
12:12:09.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.798393 30264 server_base.cc:1061] running on GCE node
12:12:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:09.800891 30298 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:09.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.802320 30264 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.812014 30264 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.813649 30264 hybrid_clock.cc:648] HybridClock initialized: now 1783512729813614 us; error 36 us; skew 500 ppm
12:12:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.814884 30264 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.822629 30264 webserver.cc:533] Webserver started at http://127.29.71.129:42507/ using document root <none> and password file <none>
12:12:09.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.823720 30264 fs_manager.cc:362] Metadata directory not provided
12:12:09.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.823974 30264 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:09.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.824466 30264 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.830363 30264 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/instance:
12:12:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "836381137da64858b3e96d2b8f127242"
12:12:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:09 on dist-test-slave-tp5p"
12:12:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.831745 30264 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal/instance:
12:12:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "836381137da64858b3e96d2b8f127242"
12:12:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:09 on dist-test-slave-tp5p"
12:12:09.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.840775 30264 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.010s	sys 0.000s
12:12:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.847565 30303 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.848856 30264 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.006s	sys 0.000s
12:12:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.849224 30264 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "836381137da64858b3e96d2b8f127242"
12:12:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:09 on dist-test-slave-tp5p"
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.849587 30264 fs_report.cc:389] FS layout report
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/data
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.896764 30264 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.898654 30264 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.899202 30264 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:09.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.902647 30264 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.908205 30264 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:12:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.908466 30264 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:12:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.908749 30264 ts_tablet_manager.cc:616] Registered 0 tablets
12:12:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:09.908995 30264 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.001s	sys 0.000s
12:12:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.097708 30264 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.129:44009
12:12:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.097856 30415 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.129:44009 every 8 connection(s)
12:12:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.101317 30264 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.111079 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 30264
12:12:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.111515 29982 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal/instance
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.122609 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/logs
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.130:0
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.130
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.130
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:10.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.163136 30416 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:10.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.163733 30416 heartbeater.cc:461] Registering TS with master...
12:12:10.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.165160 30416 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:10.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.169075 30182 ts_manager.cc:194] Registered new tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:10.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.180028 30419 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:10.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.180363 30419 heartbeater.cc:461] Registering TS with master...
12:12:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.181248 30419 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:10.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.184356 30103 ts_manager.cc:194] Registered new tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.188120 30423 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.188499 30423 heartbeater.cc:461] Registering TS with master...
12:12:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.189505 30423 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:10.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.192099 30028 ts_manager.cc:194] Registered new tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:10.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.194834 30028 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.129:46875
12:12:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:10.472980 30422 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:10.473443 30422 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:10.473850 30422 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:10.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:10.506103 30422 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:10.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:10.506961 30422 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.130
12:12:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.541493 30422 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data
12:12:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.130:0
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.130
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.130
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/logs
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.542739 30422 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:10.544973 30422 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:10.558928 30433 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:11.199154 30423 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:12.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:11.962153 30432 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 30422
12:12:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.018380 30432 kernel_stack_watchdog.cc:198] Thread 30422 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 399ms:
12:12:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
12:12:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
12:12:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
12:12:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
12:12:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.018777 30422 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.458s	user 0.558s	sys 0.864s
12:12:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.019132 30422 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.459s	user 0.558s	sys 0.864s
12:12:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:10.559717 30434 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.020027 30435 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1459 milliseconds
12:12:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.020588 30422 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.020670 30436 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.023916 30422 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:12.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.026057 30422 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:12.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.027429 30422 hybrid_clock.cc:648] HybridClock initialized: now 1783512732027390 us; error 34 us; skew 500 ppm
12:12:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.028296 30422 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.034090 30422 webserver.cc:533] Webserver started at http://127.29.71.130:42689/ using document root <none> and password file <none>
12:12:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.034993 30422 fs_manager.cc:362] Metadata directory not provided
12:12:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.035200 30422 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.035688 30422 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:12.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.039796 30422 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/instance:
12:12:12.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a40db63d1c9748e99885c3264331297a"
12:12:12.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:12 on dist-test-slave-tp5p"
12:12:12.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.040938 30422 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal/instance:
12:12:12.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a40db63d1c9748e99885c3264331297a"
12:12:12.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:12 on dist-test-slave-tp5p"
12:12:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.047725 30422 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.000s
12:12:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.053146 30445 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:12.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.054263 30422 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
12:12:12.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.054594 30422 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a40db63d1c9748e99885c3264331297a"
12:12:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:12 on dist-test-slave-tp5p"
12:12:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.054917 30422 fs_report.cc:389] FS layout report
12:12:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/data
12:12:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:12.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.107488 30422 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.109405 30422 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.109936 30422 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:12.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.113178 30422 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:12.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.118463 30422 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:12:12.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.118716 30422 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:12:12.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.118984 30422 ts_tablet_manager.cc:616] Registered 0 tablets
12:12:12.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.119189 30422 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:12:12.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.256294 30422 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.130:36197
12:12:12.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.256395 30557 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.130:36197 every 8 connection(s)
12:12:12.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.258833 30422 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:12.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.269088 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 30422
12:12:12.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.269714 29982 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal/instance
12:12:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.285892 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data
12:12:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/logs
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.131:0
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.131
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.131
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:12.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:12.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.305313 30558 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.305804 30558 heartbeater.cc:461] Registering TS with master...
12:12:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.306699 30559 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.307044 30559 heartbeater.cc:461] Registering TS with master...
12:12:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.307166 30558 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.307271 30561 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.307586 30561 heartbeater.cc:461] Registering TS with master...
12:12:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.307909 30559 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:12.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.308431 30561 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.310636 30182 ts_manager.cc:194] Registered new tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.310770 30028 ts_manager.cc:194] Registered new tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.312247 30103 ts_manager.cc:194] Registered new tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.312497 30028 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.130:43427
12:12:12.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.596352 30568 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.596896 30568 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.597587 30568 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:12.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.628858 30568 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:12.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.629855 30568 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.131
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.664592 30568 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.131:0
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.131
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.131
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/logs
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:12.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:12.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:12.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:12.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.666328 30568 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:12.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:12.668149 30568 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:12.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.679771 30574 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:13.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.316918 30561 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:13.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:12.681499 30575 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:13.793989 30577 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:13.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:13.798208 30576 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1116 milliseconds
12:12:13.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.798295 30568 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.799549 30568 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:13.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.801914 30568 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:13.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.803260 30568 hybrid_clock.cc:648] HybridClock initialized: now 1783512733803214 us; error 59 us; skew 500 ppm
12:12:13.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.804101 30568 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:13.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.810328 30568 webserver.cc:533] Webserver started at http://127.29.71.131:33861/ using document root <none> and password file <none>
12:12:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.811213 30568 fs_manager.cc:362] Metadata directory not provided
12:12:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.811426 30568 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.811952 30568 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.816170 30568 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/instance:
12:12:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c7651a898f6142c793a606c219090c09"
12:12:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:13 on dist-test-slave-tp5p"
12:12:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.817381 30568 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal/instance:
12:12:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c7651a898f6142c793a606c219090c09"
12:12:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:13 on dist-test-slave-tp5p"
12:12:13.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.824379 30568 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.000s
12:12:13.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.829797 30585 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:13.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.830907 30568 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.002s
12:12:13.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.831377 30568 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:13.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c7651a898f6142c793a606c219090c09"
12:12:13.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:13 on dist-test-slave-tp5p"
12:12:13.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.831817 30568 fs_report.cc:389] FS layout report
12:12:13.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:13.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/data
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.880914 30568 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.882514 30568 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.882995 30568 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:13.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.885686 30568 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:13.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.889760 30568 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:12:13.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.889956 30568 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:12:13.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.890249 30568 ts_tablet_manager.cc:616] Registered 0 tablets
12:12:13.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:13.890426 30568 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:12:14.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.027204 30568 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.131:38005
12:12:14.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.027364 30697 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.131:38005 every 8 connection(s)
12:12:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.029803 30568 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.037539 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 30568
12:12:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.038005 29982 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal/instance
12:12:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.063558 30700 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.064143 30700 heartbeater.cc:461] Registering TS with master...
12:12:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.064756 30698 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.065101 30698 heartbeater.cc:461] Registering TS with master...
12:12:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.065456 30700 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.065973 30698 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.065986 30699 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.066370 30699 heartbeater.cc:461] Registering TS with master...
12:12:14.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.067184 30699 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:14.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.068153 30028 ts_manager.cc:194] Registered new tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:14.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.068382 30182 ts_manager.cc:194] Registered new tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:14.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.070420 30028 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.131:55919
12:12:14.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.070358 30103 ts_manager.cc:194] Registered new tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:14.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.081151 29982 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
12:12:14.094 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
12:12:14.095 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

12:12:14.104 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.29.71.190"
      port: 46473
    }
    bound_http_address {
      host: "127.29.71.190"
      port: 43239
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.29.71.189"
      port: 40761
    }
    bound_http_address {
      host: "127.29.71.189"
      port: 35303
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.29.71.188"
      port: 37701
    }
    bound_http_address {
      host: "127.29.71.188"
      port: 37145
    }
  }
}

12:12:14.107 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

12:12:14.110 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.29.71.129"
      port: 44009
    }
    bound_http_address {
      host: "127.29.71.129"
      port: 42507
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.29.71.130"
      port: 36197
    }
    bound_http_address {
      host: "127.29.71.130"
      port: 42689
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.29.71.131"
      port: 38005
    }
    bound_http_address {
      host: "127.29.71.131"
      port: 33861
    }
  }
}

12:12:14.110 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
12:12:14.290 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
12:12:14.352 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
12:12:14.513 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.71.190:46473(127.29.71.190:46473)] connecting to peer
12:12:14.563 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.71.189:40761(127.29.71.189:40761)] connecting to peer
12:12:14.569 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.71.188:37701(127.29.71.188:37701)] connecting to peer
12:12:14.587 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.29.71.188:37701(127.29.71.188:37701)] Successfully connected to peer
12:12:14.597 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.29.71.189:40761(127.29.71.189:40761)] Successfully connected to peer
12:12:14.593 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.71.190:46473(127.29.71.190:46473)] Successfully connected to peer
12:12:14.676 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.71.190
12:12:14.679 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.71.189
12:12:14.680 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.71.188
12:12:14.891 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xeed3f397, L:/127.0.0.1:33758 - R:/127.29.71.190:46473] using SASL/PLAIN
12:12:14.891 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xb6308fc8, L:/127.0.0.1:34610 - R:/127.29.71.189:40761] using SASL/PLAIN
12:12:14.891 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x40061375, L:/127.0.0.1:59324 - R:/127.29.71.188:37701] using SASL/PLAIN
12:12:14.926 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
12:12:14.927 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=452), Traces: [0ms] refreshing cache from master, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.71.190:46473, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.71.189:40761, [90ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.71.188:37701, [427ms] Sub RPC ConnectToMaster: received response from server master-127.29.71.189:40761: OK, [427ms] Sub RPC ConnectToMaster: received response from server master-127.29.71.188:37701: OK, [427ms] Sub RPC ConnectToMaster: received response from server master-127.29.71.190:46473: OK, deferred=null) after lookup
12:12:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:14.973136 30028 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33758:
12:12:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461"
12:12:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
12:12:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:12:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
12:12:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:12:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
12:12:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
12:12:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
12:12:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:14.977348 30028 catalog_manager.cc:7077] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
12:12:15.055 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 4b97ef0f29114a6798a6dc8f5788c923
12:12:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.073341 30700 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.072751 30351 tablet_service.cc:1511] Processing CreateTablet for tablet 6a1f398f09354ae5af27df2891707f0c (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461 [id=4b97ef0f29114a6798a6dc8f5788c923]), partition=RANGE (key) PARTITION UNBOUNDED
12:12:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.074510 30351 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6a1f398f09354ae5af27df2891707f0c. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:15.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.122917 30633 tablet_service.cc:1511] Processing CreateTablet for tablet 6a1f398f09354ae5af27df2891707f0c (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461 [id=4b97ef0f29114a6798a6dc8f5788c923]), partition=RANGE (key) PARTITION UNBOUNDED
12:12:15.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.124922 30633 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6a1f398f09354ae5af27df2891707f0c. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.130287 30493 tablet_service.cc:1511] Processing CreateTablet for tablet 6a1f398f09354ae5af27df2891707f0c (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461 [id=4b97ef0f29114a6798a6dc8f5788c923]), partition=RANGE (key) PARTITION UNBOUNDED
12:12:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.132422 30493 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6a1f398f09354ae5af27df2891707f0c. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:15.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.156576 30733 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap starting.
12:12:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.167362 30733 tablet_bootstrap.cc:654] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Neither blocks nor log segments found. Creating new log.
12:12:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.171363 30734 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Bootstrap starting.
12:12:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.172122 30733 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Log is configured to *not* fsync() on all Append() calls
12:12:15.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.176846 30735 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Bootstrap starting.
12:12:15.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.178848 30734 tablet_bootstrap.cc:654] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Neither blocks nor log segments found. Creating new log.
12:12:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.189725 30733 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: No bootstrap required, opened a new log
12:12:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.190228 30733 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Time spent bootstrapping tablet: real 0.034s	user 0.015s	sys 0.000s
12:12:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.190353 30734 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Log is configured to *not* fsync() on all Append() calls
12:12:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.197808 30735 tablet_bootstrap.cc:654] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Neither blocks nor log segments found. Creating new log.
12:12:15.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.200743 30735 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Log is configured to *not* fsync() on all Append() calls
12:12:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.208181 30734 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: No bootstrap required, opened a new log
12:12:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.208719 30734 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Time spent bootstrapping tablet: real 0.038s	user 0.010s	sys 0.005s
12:12:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.224534 30735 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: No bootstrap required, opened a new log
12:12:15.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.225373 30735 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Time spent bootstrapping tablet: real 0.049s	user 0.017s	sys 0.011s
12:12:15.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.225169 30733 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:15.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.226009 30733 raft_consensus.cc:385] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:15.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.226326 30733 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 836381137da64858b3e96d2b8f127242, State: Initialized, Role: FOLLOWER
12:12:15.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.227255 30733 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:15.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.241429 30734 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:15.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.242643 30734 raft_consensus.cc:385] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:15.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.243180 30734 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a40db63d1c9748e99885c3264331297a, State: Initialized, Role: FOLLOWER
12:12:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.244849 30734 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:15.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.246411 30733 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Time spent starting tablet: real 0.056s	user 0.030s	sys 0.015s
12:12:15.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.274029 30734 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Time spent starting tablet: real 0.065s	user 0.034s	sys 0.023s
12:12:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:15.276515 30565 tablet.cc:2406] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:12:15.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.282267 30739 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:12:15.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.282866 30739 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.284034 30735 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.285089 30735 raft_consensus.cc:385] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.285550 30735 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c7651a898f6142c793a606c219090c09, State: Initialized, Role: FOLLOWER
12:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.286108 30739 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.286726 30735 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.317521 30735 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Time spent starting tablet: real 0.092s	user 0.028s	sys 0.030s
12:12:15.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.318967 30653 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09" is_pre_election: true
12:12:15.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.319844 30653 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 836381137da64858b3e96d2b8f127242 in term 0.
12:12:15.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.324702 30304 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 836381137da64858b3e96d2b8f127242, c7651a898f6142c793a606c219090c09; no voters: 
12:12:15.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.325896 30739 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 0 FOLLOWER]: Leader pre-election won for term 1
12:12:15.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.326254 30739 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
12:12:15.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.326561 30739 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 0 FOLLOWER]: Advancing to term 1
12:12:15.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.336206 30739 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:15.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.338091 30739 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 1 election: Requested vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:15.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.340003 30653 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09"
12:12:15.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.340493 30653 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 0 FOLLOWER]: Advancing to term 1
12:12:15.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.343776 30513 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a40db63d1c9748e99885c3264331297a" is_pre_election: true
12:12:15.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.344007 30512 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a40db63d1c9748e99885c3264331297a"
12:12:15.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.344666 30513 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 836381137da64858b3e96d2b8f127242 in term 0.
12:12:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.347743 30653 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 836381137da64858b3e96d2b8f127242 in term 1.
12:12:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.348697 30304 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, c7651a898f6142c793a606c219090c09; no voters: a40db63d1c9748e99885c3264331297a
12:12:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.349457 30739 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 1 FOLLOWER]: Leader election won for term 1
12:12:15.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.351243 30739 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 1 LEADER]: Becoming Leader. State: Replica: 836381137da64858b3e96d2b8f127242, State: Running, Role: LEADER
12:12:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.352126 30739 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:15.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.363174 30027 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 reported cstate change: term changed from 0 to 1, leader changed from <none> to 836381137da64858b3e96d2b8f127242 (127.29.71.129). New cstate: current_term: 1 leader_uuid: "836381137da64858b3e96d2b8f127242" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: UNKNOWN } } }
12:12:15.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:15.408959 30424 tablet.cc:2406] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:12:15.504 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 4b97ef0f29114a6798a6dc8f5788c923
12:12:15.521 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:15.526 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:15.527 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=null) after lookup
12:12:15.537 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:15.539 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] Successfully connected to peer
12:12:15.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:15.547134 30701 tablet.cc:2406] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:12:15.604 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x55bfc3aa, L:/127.0.0.1:59226 - R:/127.29.71.129:44009] using TOKEN/null
12:12:15.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.617738 30513 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 0 FOLLOWER]: Advancing to term 1
12:12:15.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.619035 30653 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 1 FOLLOWER]: Refusing update from remote peer 836381137da64858b3e96d2b8f127242: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:12:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.620535 30744 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
12:12:15.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.623916 30513 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 1 FOLLOWER]: Refusing update from remote peer 836381137da64858b3e96d2b8f127242: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:12:15.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.625188 30744 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:12:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.640237 30748 mvcc.cc:204] Tried to move back new op lower bound from 7305268165076086784 to 7305268164010426368. Current Snapshot: MvccSnapshot[applied={T|T < 7305268165076086784}]
12:12:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.643782 30749 mvcc.cc:204] Tried to move back new op lower bound from 7305268165076086784 to 7305268164010426368. Current Snapshot: MvccSnapshot[applied={T|T < 7305268165076086784}]
12:12:15.774 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:15.775 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.71.129:44009
12:12:15.776 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:15.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:15.777189 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 30264
12:12:15.813 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:15.814 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] handling channelInactive
12:12:15.815 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state READY due to: connection closed
12:12:15.815 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: connection closed
12:12:15.841 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:15.841 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:15.842 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [27ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@281060149(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@29bbafcb -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ebc067a -> wakeup thread Time-limited test)) after lookup
12:12:15.842 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:15.845 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.845 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.845 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.860 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:15.860 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:15.861 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [27ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [28ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [31ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [31ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@281060149(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@29bbafcb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ebc067a -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258))) after lookup
12:12:15.862 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:15.864 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.864 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.864 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:15.880 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:15.881 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=66), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [27ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [28ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [31ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [31ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [47ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [50ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [50ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [66ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@281060149(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@29bbafcb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ebc067a -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784))) after lookup
12:12:15.882 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:15.883 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.883 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.883 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.941 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:15.941 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:15.942 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=127), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [27ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [28ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [31ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [31ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [47ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [50ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [50ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [66ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [67ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [69ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [69ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [122ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [126ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@281060149(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@29bbafcb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ebc067a -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826))) after lookup
12:12:15.942 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:15.944 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.944 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:15.944 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:16.000 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:16.001 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:16.001 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=187), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [27ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [28ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [31ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [31ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [47ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [50ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [50ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [66ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [67ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [69ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [69ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [122ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [126ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [128ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [130ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [130ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [182ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [186ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@281060149(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@29bbafcb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826) -> (continuation of Deferred@373587502 after retry RPC@923327033), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ebc067a -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826) -> (continuation of Deferred@373587502 after retry RPC@923327033))) after lookup
12:12:16.002 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:16.003 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:16.003 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:16.003 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:17.000 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:17.001 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:17.002 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1187), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [27ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [28ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [31ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [31ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [47ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [50ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [50ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [66ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [67ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [69ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [69ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [122ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [126ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [128ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [130ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [130ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [182ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [186ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [188ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [189ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [189ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1182ms] refreshing cache from master, [1182ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1186ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@281060149(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@29bbafcb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826) -> (continuation of Deferred@373587502 after retry RPC@923327033) -> (continuation of Deferred@393907966 after retry RPC@1505119673), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ebc067a -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826) -> (continuation of Deferred@373587502 after retry RPC@923327033) -> (continuation of Deferred@393907966 after retry RPC@1505119673))) after lookup
12:12:17.003 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:17.004 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:17.004 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:17.004 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:17.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.214852 30762 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 836381137da64858b3e96d2b8f127242)
12:12:17.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.215310 30763 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 836381137da64858b3e96d2b8f127242)
12:12:17.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.215276 30762 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:17.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.215687 30763 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.217968 30763 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.219126 30762 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:17.227125 30588 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:17.236251 30448 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:17.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:17.241667 30588 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:17.245818 30448 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.246409 30653 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "a40db63d1c9748e99885c3264331297a" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09" is_pre_election: true
12:12:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.246531 30513 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "c7651a898f6142c793a606c219090c09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a40db63d1c9748e99885c3264331297a" is_pre_election: true
12:12:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.246982 30653 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a40db63d1c9748e99885c3264331297a in term 1.
12:12:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.247037 30513 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c7651a898f6142c793a606c219090c09 in term 1.
12:12:17.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.248106 30586 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09; no voters: 836381137da64858b3e96d2b8f127242
12:12:17.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.248214 30446 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09; no voters: 836381137da64858b3e96d2b8f127242
12:12:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.248761 30762 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 1 FOLLOWER]: Leader pre-election won for term 2
12:12:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.248912 30763 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 1 FOLLOWER]: Leader pre-election won for term 2
12:12:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.249099 30762 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 836381137da64858b3e96d2b8f127242)
12:12:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.249188 30763 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 1 FOLLOWER]: Starting leader election (detected failure of leader 836381137da64858b3e96d2b8f127242)
12:12:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.249413 30763 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 1 FOLLOWER]: Advancing to term 2
12:12:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.249423 30762 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 1 FOLLOWER]: Advancing to term 2
12:12:17.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.253371 30763 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:17.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.253466 30762 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:17.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.254889 30763 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 2 election: Requested vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:17.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.255177 30762 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 2 election: Requested vote from peers 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.255856 30653 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "a40db63d1c9748e99885c3264331297a" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09"
12:12:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.256516 30653 raft_consensus.cc:2393] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate a40db63d1c9748e99885c3264331297a in current term 2: Already voted for candidate c7651a898f6142c793a606c219090c09 in this term.
12:12:17.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.257620 30513 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "c7651a898f6142c793a606c219090c09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a40db63d1c9748e99885c3264331297a"
12:12:17.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.258409 30513 raft_consensus.cc:2393] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate c7651a898f6142c793a606c219090c09 in current term 2: Already voted for candidate a40db63d1c9748e99885c3264331297a in this term.
12:12:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:17.260403 30448 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:17.260520 30588 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.260756 30448 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a40db63d1c9748e99885c3264331297a; no voters: 836381137da64858b3e96d2b8f127242, c7651a898f6142c793a606c219090c09
12:12:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.260922 30588 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c7651a898f6142c793a606c219090c09; no voters: 836381137da64858b3e96d2b8f127242, a40db63d1c9748e99885c3264331297a
12:12:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.261389 30763 raft_consensus.cc:2749] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
12:12:17.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:17.261940 30762 raft_consensus.cc:2749] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
12:12:17.380 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:17.381 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:17.381 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1567), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [27ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [28ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [31ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [31ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [47ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [50ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [50ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [66ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [67ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [69ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [69ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [122ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [126ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [128ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [130ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [130ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [182ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [186ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [188ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [189ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [189ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1182ms] refreshing cache from master, [1182ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1186ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [1188ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1190ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1190ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1562ms] refreshing cache from master, [1562ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1566ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@281060149(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@29bbafcb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826) -> (continuation of Deferred@373587502 after retry RPC@923327033) -> (continuation of Deferred@393907966 after retry RPC@1505119673) -> (continuation of Deferred@757962540 after retry RPC@1176358069), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ebc067a -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826) -> (continuation of Deferred@373587502 after retry RPC@923327033) -> (continuation of Deferred@393907966 after retry RPC@1505119673) -> (continuation of Deferred@757962540 after retry RPC@1176358069))) after lookup
12:12:17.382 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:17.383 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:17.383 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:17.383 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:18.540 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:18.540 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:18.541 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=16, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2726), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [27ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [28ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [31ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [31ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [47ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [50ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [50ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [66ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [67ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [69ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [69ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [122ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [126ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [128ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [130ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [130ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [182ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [186ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [188ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [189ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [189ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1182ms] refreshing cache from master, [1182ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1186ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [1188ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1190ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1190ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1562ms] refreshing cache from master, [1562ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1566ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [1568ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1569ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1569ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2722ms] refreshing cache from master, [2722ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [2726ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@281060149(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@29bbafcb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826) -> (continuation of Deferred@373587502 after retry RPC@923327033) -> (continuation of Deferred@393907966 after retry RPC@1505119673) -> (continuation of Deferred@757962540 after retry RPC@1176358069) -> (continuation of Deferred@2102813683 after retry RPC@1426481288), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ebc067a -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826) -> (continuation of Deferred@373587502 after retry RPC@923327033) -> (continuation of Deferred@393907966 after retry RPC@1505119673) -> (continuation of Deferred@757962540 after retry RPC@1176358069) -> (continuation of Deferred@2102813683 after retry RPC@1426481288))) after lookup
12:12:18.542 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:18.543 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:18.543 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:18.543 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:19.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.197288 30772 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:12:19.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.197674 30772 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:19.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.199249 30772 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:19.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.200192 30653 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "a40db63d1c9748e99885c3264331297a" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09" is_pre_election: true
12:12:19.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.200778 30653 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a40db63d1c9748e99885c3264331297a in term 2.
12:12:19.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.201864 30446 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09; no voters: 
12:12:19.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.202579 30772 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 2 FOLLOWER]: Leader pre-election won for term 3
12:12:19.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.202888 30772 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
12:12:19.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.203188 30772 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 2 FOLLOWER]: Advancing to term 3
12:12:19.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:19.204053 30448 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.207417 30772 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.208688 30772 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 3 election: Requested vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.209417 30653 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "a40db63d1c9748e99885c3264331297a" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09"
12:12:19.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.209798 30653 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 2 FOLLOWER]: Advancing to term 3
12:12:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:19.211889 30448 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.214450 30653 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a40db63d1c9748e99885c3264331297a in term 3.
12:12:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.215261 30446 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09; no voters: 836381137da64858b3e96d2b8f127242
12:12:19.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.215809 30772 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 3 FOLLOWER]: Leader election won for term 3
12:12:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.216713 30772 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 3 LEADER]: Becoming Leader. State: Replica: a40db63d1c9748e99885c3264331297a, State: Running, Role: LEADER
12:12:19.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.217640 30772 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.224992 30028 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a reported cstate change: term changed from 1 to 3, leader changed from 836381137da64858b3e96d2b8f127242 (127.29.71.129) to a40db63d1c9748e99885c3264331297a (127.29.71.130). New cstate: current_term: 3 leader_uuid: "a40db63d1c9748e99885c3264331297a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: HEALTHY } } }
12:12:19.500 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:19.501 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:19.501 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=18, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=3687), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [27ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [28ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [31ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [31ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [46ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [47ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [50ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [50ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [66ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [67ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [69ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [69ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [122ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [126ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [128ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [130ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [130ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [182ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [186ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [188ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [189ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [189ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1182ms] refreshing cache from master, [1182ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1186ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [1188ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1190ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1190ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1562ms] refreshing cache from master, [1562ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1566ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [1568ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1569ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1569ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2722ms] refreshing cache from master, [2722ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [2726ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [2727ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2729ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2729ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [3682ms] refreshing cache from master, [3682ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [3686ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@281060149(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@29bbafcb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826) -> (continuation of Deferred@373587502 after retry RPC@923327033) -> (continuation of Deferred@393907966 after retry RPC@1505119673) -> (continuation of Deferred@757962540 after retry RPC@1176358069) -> (continuation of Deferred@2102813683 after retry RPC@1426481288) -> (continuation of Deferred@1389403951 after retry RPC@891950959), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ebc067a -> wakeup thread Time-limited test -> (continuation of Deferred@682940953 after retry RPC@484850258) -> (continuation of Deferred@1118500745 after retry RPC@1893827784) -> (continuation of Deferred@1153431608 after retry RPC@967093826) -> (continuation of Deferred@373587502 after retry RPC@923327033) -> (continuation of Deferred@393907966 after retry RPC@1505119673) -> (continuation of Deferred@757962540 after retry RPC@1176358069) -> (continuation of Deferred@2102813683 after retry RPC@1426481288) -> (continuation of Deferred@1389403951 after retry RPC@891950959))) after lookup
12:12:19.502 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:19.503 [DEBUG - kudu-nio-4] (Connection.java:757) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] Successfully connected to peer
12:12:19.533 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0xf73027e9, L:/127.0.0.1:55300 - R:/127.29.71.130:36197] using TOKEN/null
12:12:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.545346 30653 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 3 FOLLOWER]: Refusing update from remote peer a40db63d1c9748e99885c3264331297a: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
12:12:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:19.547209 30448 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a -> Peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Couldn't send request to peer 836381137da64858b3e96d2b8f127242. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.547451 30775 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [LEADER]: Connected to new peer: Peer: permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
12:12:19.600 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.71.129:44009
12:12:19.601 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.603013 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/logs
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.129:44009
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.129
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42507
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.129
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:19.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:19.897598 30795 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:19.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:19.898064 30795 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:19.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:19.898515 30795 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:19.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:19.930080 30795 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:19.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:19.930884 30795 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.129
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.967981 30795 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.129:44009
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.129
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42507
12:12:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.129
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/logs
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.969314 30795 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:19.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:19.970933 30795 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:19.982928 30801 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:19.983594 30802 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:21.276253 30804 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:21.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:21.279152 30795 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.296s	user 0.483s	sys 0.785s
12:12:21.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:21.279636 30795 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.297s	user 0.484s	sys 0.786s
12:12:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:21.279775 30803 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1290 milliseconds
12:12:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.279922 30795 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.281427 30795 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.283993 30795 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.285538 30795 hybrid_clock.cc:648] HybridClock initialized: now 1783512741285493 us; error 42 us; skew 500 ppm
12:12:21.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.286633 30795 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:21.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.296366 30795 webserver.cc:533] Webserver started at http://127.29.71.129:42507/ using document root <none> and password file <none>
12:12:21.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.297596 30795 fs_manager.cc:362] Metadata directory not provided
12:12:21.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.297880 30795 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.307991 30795 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.000s
12:12:21.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.313582 30813 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:21.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.314900 30795 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
12:12:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.315280 30795 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "836381137da64858b3e96d2b8f127242"
12:12:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:09 on dist-test-slave-tp5p"
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.317878 30795 fs_report.cc:389] FS layout report
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/data
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.398723 30795 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.400501 30795 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.400983 30795 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.403473 30795 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:21.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.409842 30821 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:12:21.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.417477 30795 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:21.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.417711 30795 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.000s	sys 0.002s
12:12:21.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.418031 30795 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:21.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.422616 30795 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:21.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.422807 30795 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
12:12:21.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.423113 30821 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap starting.
12:12:21.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.480490 30821 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Log is configured to *not* fsync() on all Append() calls
12:12:21.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.510284 30821 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:21.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.511080 30821 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap complete.
12:12:21.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.512581 30821 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Time spent bootstrapping tablet: real 0.090s	user 0.055s	sys 0.027s
12:12:21.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.533432 30821 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:21.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.535744 30821 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 836381137da64858b3e96d2b8f127242, State: Initialized, Role: FOLLOWER
12:12:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.536799 30821 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:21.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.540699 30821 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Time spent starting tablet: real 0.028s	user 0.022s	sys 0.009s
12:12:21.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.579538 30795 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.129:44009
12:12:21.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.579805 30931 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.129:44009 every 8 connection(s)
12:12:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.582507 30795 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.585191 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 30795
12:12:21.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:21.625 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:21.627 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.71.130:36197
12:12:21.628 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:12:21.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.628405 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 30422
12:12:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.636579 30932 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.637116 30932 heartbeater.cc:461] Registering TS with master...
12:12:21.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.638521 30932 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.643373 30182 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:21.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.645289 30933 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:21.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.645639 30933 heartbeater.cc:461] Registering TS with master...
12:12:21.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.646517 30933 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:21.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.649746 30102 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:21.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.651229 30935 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:21.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.651563 30935 heartbeater.cc:461] Registering TS with master...
12:12:21.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.652369 30935 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:21.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.655535 30028 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:21.658733 30028 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.129:42197
12:12:21.664 [DEBUG - kudu-nio-4] (Connection.java:239) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] handling channelInactive
12:12:21.664 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state READY due to: connection closed
12:12:21.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:21.665 [DEBUG - Time-limited test] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:21.666 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.666 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.666 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.681 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:21.681 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:21.681 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [1ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@616813204(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@4137380c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3133f954 -> wakeup thread Time-limited test)) after lookup
12:12:21.682 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:21.683 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.683 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.683 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.700 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:21.700 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:21.700 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [1ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [17ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [18ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [18ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@616813204(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@4137380c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@79127609 after retry RPC@478861715), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3133f954 -> wakeup thread Time-limited test -> (continuation of Deferred@79127609 after retry RPC@478861715))) after lookup
12:12:21.701 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:21.703 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.703 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.703 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.740 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:21.741 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:21.741 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=76), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [1ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [17ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [18ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [18ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [36ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [38ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [38ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [75ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@616813204(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@4137380c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@79127609 after retry RPC@478861715) -> (continuation of Deferred@211531083 after retry RPC@2034295301), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3133f954 -> wakeup thread Time-limited test -> (continuation of Deferred@79127609 after retry RPC@478861715) -> (continuation of Deferred@211531083 after retry RPC@2034295301))) after lookup
12:12:21.742 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:21.743 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.743 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.743 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.820 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:21.820 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:21.821 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=156), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [1ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [17ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [18ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [18ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [36ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [38ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [38ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [75ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [77ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [78ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [78ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [155ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@616813204(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@4137380c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@79127609 after retry RPC@478861715) -> (continuation of Deferred@211531083 after retry RPC@2034295301) -> (continuation of Deferred@1834804732 after retry RPC@774395958), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3133f954 -> wakeup thread Time-limited test -> (continuation of Deferred@79127609 after retry RPC@478861715) -> (continuation of Deferred@211531083 after retry RPC@2034295301) -> (continuation of Deferred@1834804732 after retry RPC@774395958))) after lookup
12:12:21.822 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:21.823 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.823 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:21.823 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:22.021 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:22.021 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:22.022 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=356), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [1ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [17ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [18ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [18ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [36ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [38ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [38ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [75ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [77ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [78ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [78ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [155ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [156ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [158ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [158ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [352ms] refreshing cache from master, [352ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [356ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@616813204(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@4137380c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@79127609 after retry RPC@478861715) -> (continuation of Deferred@211531083 after retry RPC@2034295301) -> (continuation of Deferred@1834804732 after retry RPC@774395958) -> (continuation of Deferred@1557842345 after retry RPC@819975148), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3133f954 -> wakeup thread Time-limited test -> (continuation of Deferred@79127609 after retry RPC@478861715) -> (continuation of Deferred@211531083 after retry RPC@2034295301) -> (continuation of Deferred@1834804732 after retry RPC@774395958) -> (continuation of Deferred@1557842345 after retry RPC@819975148))) after lookup
12:12:22.022 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:22.023 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:22.023 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:22.024 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:22.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.634191 30943 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader a40db63d1c9748e99885c3264331297a)
12:12:22.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.634820 30943 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:22.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.637135 30943 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:22.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:22.640182 30586 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111) [suppressed 1 similar messages]
12:12:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:22.643563 30586 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111)
12:12:22.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.648499 30886 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "c7651a898f6142c793a606c219090c09" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "836381137da64858b3e96d2b8f127242" is_pre_election: true
12:12:22.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.649286 30886 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c7651a898f6142c793a606c219090c09 in term 1.
12:12:22.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.650653 30588 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, c7651a898f6142c793a606c219090c09; no voters: a40db63d1c9748e99885c3264331297a
12:12:22.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.651376 30943 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 3 FOLLOWER]: Leader pre-election won for term 4
12:12:22.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.651624 30943 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 3 FOLLOWER]: Starting leader election (detected failure of leader a40db63d1c9748e99885c3264331297a)
12:12:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.651935 30943 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 3 FOLLOWER]: Advancing to term 4
12:12:22.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.656963 30943 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.658325 30943 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 4 election: Requested vote from peers 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.658927 30886 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "c7651a898f6142c793a606c219090c09" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "836381137da64858b3e96d2b8f127242"
12:12:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.659339 30886 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 1 FOLLOWER]: Advancing to term 4
12:12:22.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:22.661379 30586 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111)
12:12:22.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.662247 30935 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:22.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.665625 30886 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c7651a898f6142c793a606c219090c09 in term 4.
12:12:22.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.666467 30588 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, c7651a898f6142c793a606c219090c09; no voters: a40db63d1c9748e99885c3264331297a
12:12:22.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.666991 30943 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 4 FOLLOWER]: Leader election won for term 4
12:12:22.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.668126 30943 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 4 LEADER]: Becoming Leader. State: Replica: c7651a898f6142c793a606c219090c09, State: Running, Role: LEADER
12:12:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.669237 30943 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:22.678397 30028 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 reported cstate change: term changed from 3 to 4, leader changed from a40db63d1c9748e99885c3264331297a (127.29.71.130) to c7651a898f6142c793a606c219090c09 (127.29.71.131). New cstate: current_term: 4 leader_uuid: "c7651a898f6142c793a606c219090c09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: UNKNOWN } } }
12:12:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:23.227101 30586 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 -> Peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Couldn't send request to peer a40db63d1c9748e99885c3264331297a. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:23.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:23.235252 30886 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 4 FOLLOWER]: Refusing update from remote peer c7651a898f6142c793a606c219090c09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 4 index: 9. (index mismatch)
12:12:23.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:23.236562 30961 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
12:12:23.680 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:23.681 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:23.681 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=2016), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [1ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [15ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [17ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [18ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [18ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [36ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [38ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [38ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [75ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [77ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [78ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [78ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [155ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [156ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [158ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [158ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [352ms] refreshing cache from master, [352ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [356ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [357ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [359ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [359ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2011ms] refreshing cache from master, [2012ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [2015ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@616813204(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@4137380c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@79127609 after retry RPC@478861715) -> (continuation of Deferred@211531083 after retry RPC@2034295301) -> (continuation of Deferred@1834804732 after retry RPC@774395958) -> (continuation of Deferred@1557842345 after retry RPC@819975148) -> (continuation of Deferred@1967218317 after retry RPC@1003467642), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3133f954 -> wakeup thread Time-limited test -> (continuation of Deferred@79127609 after retry RPC@478861715) -> (continuation of Deferred@211531083 after retry RPC@2034295301) -> (continuation of Deferred@1834804732 after retry RPC@774395958) -> (continuation of Deferred@1557842345 after retry RPC@819975148) -> (continuation of Deferred@1967218317 after retry RPC@1003467642))) after lookup
12:12:23.682 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:23.683 [DEBUG - kudu-nio-3] (Connection.java:757) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] Successfully connected to peer
12:12:23.706 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x5e84f6ea, L:/127.0.0.1:43610 - R:/127.29.71.131:38005] using TOKEN/null
12:12:23.750 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.71.130:36197
12:12:23.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:12:23.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:23.752482 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/logs
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.130:36197
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.130
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42689
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.130
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:24.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:24.064244 30975 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:24.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:24.064767 30975 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:24.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:24.065397 30975 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:24.101967 30975 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:24.102926 30975 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.130
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:24.141618 30975 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.130:36197
12:12:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.130
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42689
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.130
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/logs
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:24.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:24.143455 30975 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:24.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:24.145761 30975 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:24.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:24.160002 30982 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:25.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:24.161255 30983 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:25.553514 30984 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1387 milliseconds
12:12:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:25.554733 30985 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:25.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:25.557070 30975 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.397s	user 0.512s	sys 0.880s
12:12:25.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:25.557329 30975 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.398s	user 0.512s	sys 0.880s
12:12:25.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.557617 30975 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:25.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.558723 30975 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:25.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.560921 30975 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:25.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.562279 30975 hybrid_clock.cc:648] HybridClock initialized: now 1783512745562214 us; error 86 us; skew 500 ppm
12:12:25.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.563126 30975 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:25.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.570652 30975 webserver.cc:533] Webserver started at http://127.29.71.130:42689/ using document root <none> and password file <none>
12:12:25.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.571753 30975 fs_manager.cc:362] Metadata directory not provided
12:12:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.571985 30975 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:25.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.580438 30975 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.000s
12:12:25.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.586088 30995 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.587296 30975 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.005s
12:12:25.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.587788 30975 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:25.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a40db63d1c9748e99885c3264331297a"
12:12:25.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:12 on dist-test-slave-tp5p"
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.590700 30975 fs_report.cc:389] FS layout report
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/data
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:25.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.658099 30975 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.659570 30975 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:25.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.659986 30975 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.662544 30975 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:25.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.668427 31002 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:12:25.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.676230 30975 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:25.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.676471 30975 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.001s	sys 0.001s
12:12:25.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.676805 30975 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.681363 30975 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.681543 30975 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
12:12:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.681864 31002 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Bootstrap starting.
12:12:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:25.697412 30586 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 -> Peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Couldn't send request to peer a40db63d1c9748e99885c3264331297a. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
12:12:25.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.737473 31002 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Log is configured to *not* fsync() on all Append() calls
12:12:25.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.776630 31002 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:25.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.777762 31002 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Bootstrap complete.
12:12:25.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.779177 31002 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Time spent bootstrapping tablet: real 0.098s	user 0.068s	sys 0.025s
12:12:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.803699 31002 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.807062 31002 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: a40db63d1c9748e99885c3264331297a, State: Initialized, Role: FOLLOWER
12:12:25.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.808494 31002 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:25.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.813248 31002 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Time spent starting tablet: real 0.034s	user 0.035s	sys 0.000s
12:12:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.857319 30975 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.130:36197
12:12:25.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.857700 31112 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.130:36197 every 8 connection(s)
12:12:25.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.859651 30975 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:25.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.867434 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 30975
12:12:25.868 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.897290 31114 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.897768 31114 heartbeater.cc:461] Registering TS with master...
12:12:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.898962 31114 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:25.902 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:25.903 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.71.131:38005
12:12:25.904 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:25.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.904495 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 30568
12:12:25.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.904951 30102 ts_manager.cc:194] Re-registered known tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:25.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.909314 31115 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:25.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.909452 31113 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:25.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.909667 31115 heartbeater.cc:461] Registering TS with master...
12:12:25.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.909734 31113 heartbeater.cc:461] Registering TS with master...
12:12:25.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.910647 31115 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:25.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.910614 31113 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:25.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.913520 30028 ts_manager.cc:194] Re-registered known tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:25.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.913792 30182 ts_manager.cc:194] Re-registered known tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:25.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:25.916147 30028 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.130:46779
12:12:25.928 [DEBUG - kudu-nio-3] (Connection.java:239) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] handling channelInactive
12:12:25.928 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state READY due to: connection closed
12:12:25.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:25.929 [DEBUG - Time-limited test] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:25.930 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.930 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.931 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.940 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:25.940 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:25.941 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=11), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@793675801(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@695cbae9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a5fee5f -> wakeup thread Time-limited test)) after lookup
12:12:25.941 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:25.942 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.942 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.943 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.959 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:25.960 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:25.960 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server c7651a898f6142c793a606c219090c09, [14ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [14ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@793675801(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@695cbae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a5fee5f -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056))) after lookup
12:12:25.961 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:25.961 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.962 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.962 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.980 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:25.980 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:25.980 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=51), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server c7651a898f6142c793a606c219090c09, [14ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [14ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [31ms] sending RPC to server c7651a898f6142c793a606c219090c09, [33ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [33ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [47ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [51ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@793675801(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@695cbae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056) -> (continuation of Deferred@1990663402 after retry RPC@1296789731), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a5fee5f -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056) -> (continuation of Deferred@1990663402 after retry RPC@1296789731))) after lookup
12:12:25.981 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:25.982 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.982 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:25.982 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:26.081 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:26.081 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:26.081 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=152), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server c7651a898f6142c793a606c219090c09, [14ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [14ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [31ms] sending RPC to server c7651a898f6142c793a606c219090c09, [33ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [33ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [47ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [51ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [51ms] sending RPC to server c7651a898f6142c793a606c219090c09, [53ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [53ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [148ms] refreshing cache from master, [148ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [151ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@793675801(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@695cbae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056) -> (continuation of Deferred@1990663402 after retry RPC@1296789731) -> (continuation of Deferred@506554776 after retry RPC@141283029), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a5fee5f -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056) -> (continuation of Deferred@1990663402 after retry RPC@1296789731) -> (continuation of Deferred@506554776 after retry RPC@141283029))) after lookup
12:12:26.082 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:26.083 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:26.083 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:26.083 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:26.160 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:26.161 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:26.161 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=232), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server c7651a898f6142c793a606c219090c09, [14ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [14ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [31ms] sending RPC to server c7651a898f6142c793a606c219090c09, [33ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [33ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [47ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [51ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [51ms] sending RPC to server c7651a898f6142c793a606c219090c09, [53ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [53ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [148ms] refreshing cache from master, [148ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [151ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [152ms] sending RPC to server c7651a898f6142c793a606c219090c09, [154ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [154ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [228ms] refreshing cache from master, [228ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [231ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@793675801(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@695cbae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056) -> (continuation of Deferred@1990663402 after retry RPC@1296789731) -> (continuation of Deferred@506554776 after retry RPC@141283029) -> (continuation of Deferred@1921293615 after retry RPC@1193754326), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a5fee5f -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056) -> (continuation of Deferred@1990663402 after retry RPC@1296789731) -> (continuation of Deferred@506554776 after retry RPC@141283029) -> (continuation of Deferred@1921293615 after retry RPC@1193754326))) after lookup
12:12:26.161 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:26.162 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:26.162 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:26.163 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:26.740 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:26.741 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:26.741 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=812), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server c7651a898f6142c793a606c219090c09, [14ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [14ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [31ms] sending RPC to server c7651a898f6142c793a606c219090c09, [33ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [33ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [47ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [51ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [51ms] sending RPC to server c7651a898f6142c793a606c219090c09, [53ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [53ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [148ms] refreshing cache from master, [148ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [151ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [152ms] sending RPC to server c7651a898f6142c793a606c219090c09, [154ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [154ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [228ms] refreshing cache from master, [228ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [231ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [232ms] sending RPC to server c7651a898f6142c793a606c219090c09, [234ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [234ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [807ms] refreshing cache from master, [808ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [811ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@793675801(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@695cbae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056) -> (continuation of Deferred@1990663402 after retry RPC@1296789731) -> (continuation of Deferred@506554776 after retry RPC@141283029) -> (continuation of Deferred@1921293615 after retry RPC@1193754326) -> (continuation of Deferred@483835216 after retry RPC@1660543378), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a5fee5f -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056) -> (continuation of Deferred@1990663402 after retry RPC@1296789731) -> (continuation of Deferred@506554776 after retry RPC@141283029) -> (continuation of Deferred@1921293615 after retry RPC@1193754326) -> (continuation of Deferred@483835216 after retry RPC@1660543378))) after lookup
12:12:26.742 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:26.743 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:26.743 [DEBUG - kudu-nio-2] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:26.743 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:26.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:26.918818 31115 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:27.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.009213 31124 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c7651a898f6142c793a606c219090c09)
12:12:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.009598 31124 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.012164 31124 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:27.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:27.015632 30815 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111)
12:12:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:27.019625 30815 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111)
12:12:27.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.023990 31067 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "a40db63d1c9748e99885c3264331297a" is_pre_election: true
12:12:27.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.024585 31067 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 836381137da64858b3e96d2b8f127242 in term 3.
12:12:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.025758 30815 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, a40db63d1c9748e99885c3264331297a; no voters: c7651a898f6142c793a606c219090c09
12:12:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.026412 31124 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 4 FOLLOWER]: Leader pre-election won for term 5
12:12:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.026716 31124 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 4 FOLLOWER]: Starting leader election (detected failure of leader c7651a898f6142c793a606c219090c09)
12:12:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.026962 31124 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 4 FOLLOWER]: Advancing to term 5
12:12:27.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.031314 31124 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.032848 31124 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 5 election: Requested vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.034914 31067 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "a40db63d1c9748e99885c3264331297a"
12:12:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.035354 31067 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 3 FOLLOWER]: Advancing to term 5
12:12:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:27.036222 30815 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111)
12:12:27.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.041456 31067 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 836381137da64858b3e96d2b8f127242 in term 5.
12:12:27.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.042289 30815 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, a40db63d1c9748e99885c3264331297a; no voters: c7651a898f6142c793a606c219090c09
12:12:27.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.042836 31124 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 5 FOLLOWER]: Leader election won for term 5
12:12:27.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.044237 31124 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 5 LEADER]: Becoming Leader. State: Replica: 836381137da64858b3e96d2b8f127242, State: Running, Role: LEADER
12:12:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.045423 31124 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:27.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.054607 30028 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 reported cstate change: term changed from 4 to 5, leader changed from c7651a898f6142c793a606c219090c09 (127.29.71.131) to 836381137da64858b3e96d2b8f127242 (127.29.71.129). New cstate: current_term: 5 leader_uuid: "836381137da64858b3e96d2b8f127242" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: UNKNOWN } } }
12:12:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.434007 31067 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 5 FOLLOWER]: Refusing update from remote peer 836381137da64858b3e96d2b8f127242: Log matching property violated. Preceding OpId in replica: term: 3 index: 8. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
12:12:27.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.435529 31124 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
12:12:27.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:27.449242 31143 mvcc.cc:204] Tried to move back new op lower bound from 7305268198242271232 to 7305268193981444096. Current Snapshot: MvccSnapshot[applied={T|T < 7305268198242271232}]
12:12:27.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:27.454190 30815 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 -> Peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Couldn't send request to peer c7651a898f6142c793a606c219090c09. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:28.980 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:28.980 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:28.981 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3051), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server c7651a898f6142c793a606c219090c09, [14ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [14ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [30ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [31ms] sending RPC to server c7651a898f6142c793a606c219090c09, [33ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [33ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [47ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [51ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [51ms] sending RPC to server c7651a898f6142c793a606c219090c09, [53ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [53ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [148ms] refreshing cache from master, [148ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [151ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [152ms] sending RPC to server c7651a898f6142c793a606c219090c09, [154ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [154ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [228ms] refreshing cache from master, [228ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [231ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [232ms] sending RPC to server c7651a898f6142c793a606c219090c09, [234ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [234ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [807ms] refreshing cache from master, [808ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [811ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [813ms] sending RPC to server c7651a898f6142c793a606c219090c09, [814ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [814ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [3047ms] refreshing cache from master, [3047ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [3051ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@793675801(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@695cbae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056) -> (continuation of Deferred@1990663402 after retry RPC@1296789731) -> (continuation of Deferred@506554776 after retry RPC@141283029) -> (continuation of Deferred@1921293615 after retry RPC@1193754326) -> (continuation of Deferred@483835216 after retry RPC@1660543378) -> (continuation of Deferred@431885915 after retry RPC@879135119), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a5fee5f -> wakeup thread Time-limited test -> (continuation of Deferred@2104775071 after retry RPC@1492936056) -> (continuation of Deferred@1990663402 after retry RPC@1296789731) -> (continuation of Deferred@506554776 after retry RPC@141283029) -> (continuation of Deferred@1921293615 after retry RPC@1193754326) -> (continuation of Deferred@483835216 after retry RPC@1660543378) -> (continuation of Deferred@431885915 after retry RPC@879135119))) after lookup
12:12:28.981 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:28.982 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] Successfully connected to peer
12:12:28.999 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x1b409353, L:/127.0.0.1:37376 - R:/127.29.71.129:44009] using TOKEN/null
12:12:29.044 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.71.131:38005
12:12:29.044 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:29.045661 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/logs
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.131:38005
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.131
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33861
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.131
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:29.352285 31158 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:29.352725 31158 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:29.353202 31158 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:29.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:29.384943 31158 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:29.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:29.385808 31158 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.131
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:29.422016 31158 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.131:38005
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.131
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33861
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.131
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/logs
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:29.423332 31158 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:29.425077 31158 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:29.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:29.435959 31166 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:29.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:29.755138 30815 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 -> Peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Couldn't send request to peer c7651a898f6142c793a606c219090c09. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
12:12:30.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:29.437601 31167 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:30.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:30.606431 31169 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:30.608088 31168 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1167 milliseconds
12:12:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.608211 31158 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.609488 31158 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.611939 31158 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:30.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.613343 31158 hybrid_clock.cc:648] HybridClock initialized: now 1783512750613298 us; error 45 us; skew 500 ppm
12:12:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.614099 31158 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.620015 31158 webserver.cc:533] Webserver started at http://127.29.71.131:33861/ using document root <none> and password file <none>
12:12:30.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.620833 31158 fs_manager.cc:362] Metadata directory not provided
12:12:30.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.621068 31158 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.628156 31158 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
12:12:30.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.632324 31177 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.633384 31158 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
12:12:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.633713 31158 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c7651a898f6142c793a606c219090c09"
12:12:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:13 on dist-test-slave-tp5p"
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.635516 31158 fs_report.cc:389] FS layout report
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/data
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.689965 31158 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.691483 31158 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.691977 31158 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.694571 31158 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:30.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.700172 31184 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:12:30.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.708365 31158 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:30.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.708570 31158 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.000s
12:12:30.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.708899 31158 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:30.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.713282 31158 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:30.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.713459 31158 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
12:12:30.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.713783 31184 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Bootstrap starting.
12:12:30.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.775439 31184 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Log is configured to *not* fsync() on all Append() calls
12:12:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.827044 31184 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.828027 31184 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Bootstrap complete.
12:12:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.829695 31184 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Time spent bootstrapping tablet: real 0.116s	user 0.091s	sys 0.019s
12:12:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.848898 31184 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.851482 31184 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: c7651a898f6142c793a606c219090c09, State: Initialized, Role: FOLLOWER
12:12:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.852461 31184 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:30.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.857261 31184 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Time spent starting tablet: real 0.027s	user 0.026s	sys 0.000s
12:12:30.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.885493 31158 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.131:38005
12:12:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.885825 31292 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.131:38005 every 8 connection(s)
12:12:30.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.887977 31158 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.892920 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 31158
12:12:30.893 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:30.928 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:30.935 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.71.129:44009
12:12:30.936 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:30.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.936549 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 30795
12:12:30.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.940924 31294 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:30.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.941331 31294 heartbeater.cc:461] Registering TS with master...
12:12:30.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.942813 31294 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:30.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.946537 31293 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:30.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.946986 31293 heartbeater.cc:461] Registering TS with master...
12:12:30.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.947844 31293 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.949676 31295 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.949962 31295 heartbeater.cc:461] Registering TS with master...
12:12:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.950420 30102 ts_manager.cc:194] Re-registered known tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.950837 31295 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:30.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.953658 30182 ts_manager.cc:194] Re-registered known tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:30.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.953823 30028 ts_manager.cc:194] Re-registered known tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:30.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:30.956327 30028 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.131:40857
12:12:30.970 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] handling channelInactive
12:12:30.970 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state READY due to: connection closed
12:12:30.970 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:30.971 [DEBUG - Time-limited test] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:30.972 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:30.972 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:30.972 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:30.979 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:30.980 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:30.980 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [9ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1055213609(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3ecf848b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@508badd1 -> wakeup thread Time-limited test)) after lookup
12:12:30.980 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:30.981 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:30.982 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:30.982 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:30.999 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:31.000 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:31.000 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [9ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [10ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [12ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [12ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1055213609(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3ecf848b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@508badd1 -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520))) after lookup
12:12:31.001 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:31.001 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.001 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.002 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.039 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:31.039 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:31.040 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [9ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [10ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [12ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [12ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [30ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [32ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [32ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [69ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1055213609(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3ecf848b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520) -> (continuation of Deferred@812923913 after retry RPC@81752961), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@508badd1 -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520) -> (continuation of Deferred@812923913 after retry RPC@81752961))) after lookup
12:12:31.040 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:31.041 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.041 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.042 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.060 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:31.060 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:31.060 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=90), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [9ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [10ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [12ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [12ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [30ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [32ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [32ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [69ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [70ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [72ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [72ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [86ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [90ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1055213609(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3ecf848b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520) -> (continuation of Deferred@812923913 after retry RPC@81752961) -> (continuation of Deferred@1219266070 after retry RPC@923838457), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@508badd1 -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520) -> (continuation of Deferred@812923913 after retry RPC@81752961) -> (continuation of Deferred@1219266070 after retry RPC@923838457))) after lookup
12:12:31.061 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:31.062 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.062 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.063 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.340 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:31.341 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:31.341 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=370), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [9ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [10ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [12ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [12ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [30ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [32ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [32ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [69ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [70ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [72ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [72ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [86ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [90ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [91ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [93ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [93ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [367ms] refreshing cache from master, [367ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [370ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1055213609(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3ecf848b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520) -> (continuation of Deferred@812923913 after retry RPC@81752961) -> (continuation of Deferred@1219266070 after retry RPC@923838457) -> (continuation of Deferred@1920836104 after retry RPC@243679889), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@508badd1 -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520) -> (continuation of Deferred@812923913 after retry RPC@81752961) -> (continuation of Deferred@1219266070 after retry RPC@923838457) -> (continuation of Deferred@1920836104 after retry RPC@243679889))) after lookup
12:12:31.342 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:31.343 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.343 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.343 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.921 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:31.921 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:31.921 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=951), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [9ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [10ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [12ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [12ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [30ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [32ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [32ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [69ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [70ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [72ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [72ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [86ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [90ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [91ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [93ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [93ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [367ms] refreshing cache from master, [367ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [370ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [372ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [373ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [373ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [947ms] refreshing cache from master, [947ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [950ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1055213609(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3ecf848b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520) -> (continuation of Deferred@812923913 after retry RPC@81752961) -> (continuation of Deferred@1219266070 after retry RPC@923838457) -> (continuation of Deferred@1920836104 after retry RPC@243679889) -> (continuation of Deferred@1225895117 after retry RPC@1975516596), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@508badd1 -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520) -> (continuation of Deferred@812923913 after retry RPC@81752961) -> (continuation of Deferred@1219266070 after retry RPC@923838457) -> (continuation of Deferred@1920836104 after retry RPC@243679889) -> (continuation of Deferred@1225895117 after retry RPC@1975516596))) after lookup
12:12:31.922 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:31.923 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.923 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.923 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:31.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.939955 31306 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 836381137da64858b3e96d2b8f127242)
12:12:31.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.940409 31306 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:31.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.942925 31306 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:31.946103 30998 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:31.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:31.950541 30998 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:31.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.954625 31247 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "a40db63d1c9748e99885c3264331297a" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09" is_pre_election: true
12:12:31.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.955276 31247 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a40db63d1c9748e99885c3264331297a in term 4.
12:12:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.956490 30996 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09; no voters: 836381137da64858b3e96d2b8f127242
12:12:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.957340 31306 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 5 FOLLOWER]: Leader pre-election won for term 6
12:12:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.957664 31306 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 5 FOLLOWER]: Starting leader election (detected failure of leader 836381137da64858b3e96d2b8f127242)
12:12:31.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.957973 31306 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 5 FOLLOWER]: Advancing to term 6
12:12:31.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.960705 31295 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:31.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.963327 31306 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:31.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.964870 31306 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 6 election: Requested vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:31.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.965579 31247 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "a40db63d1c9748e99885c3264331297a" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09"
12:12:31.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.966094 31247 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 4 FOLLOWER]: Advancing to term 6
12:12:31.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:31.968541 30998 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.972499 31247 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a40db63d1c9748e99885c3264331297a in term 6.
12:12:31.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.973238 30996 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09; no voters: 836381137da64858b3e96d2b8f127242
12:12:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.973765 31306 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 6 FOLLOWER]: Leader election won for term 6
12:12:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.975198 31306 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 6 LEADER]: Becoming Leader. State: Replica: a40db63d1c9748e99885c3264331297a, State: Running, Role: LEADER
12:12:31.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.975896 31306 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:31.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:31.983948 30028 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a reported cstate change: term changed from 5 to 6, leader changed from 836381137da64858b3e96d2b8f127242 (127.29.71.129) to a40db63d1c9748e99885c3264331297a (127.29.71.130). New cstate: current_term: 6 leader_uuid: "a40db63d1c9748e99885c3264331297a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: HEALTHY } } }
12:12:32.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:32.507851 31247 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 6 FOLLOWER]: Refusing update from remote peer a40db63d1c9748e99885c3264331297a: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 6 index: 17. (index mismatch)
12:12:32.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:32.509258 31323 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [LEADER]: Connected to new peer: Peer: permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
12:12:32.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:32.546530 30998 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a -> Peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Couldn't send request to peer 836381137da64858b3e96d2b8f127242. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:33.400 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:33.401 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:33.401 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2430), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [9ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [10ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [12ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [12ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [30ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [32ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [32ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [69ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [70ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [72ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [72ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [86ms] refreshing cache from master, [87ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [90ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [91ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [93ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [93ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [367ms] refreshing cache from master, [367ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [370ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [372ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [373ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [373ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [947ms] refreshing cache from master, [947ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [950ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [952ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [953ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [953ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2426ms] refreshing cache from master, [2426ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [2430ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1055213609(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3ecf848b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520) -> (continuation of Deferred@812923913 after retry RPC@81752961) -> (continuation of Deferred@1219266070 after retry RPC@923838457) -> (continuation of Deferred@1920836104 after retry RPC@243679889) -> (continuation of Deferred@1225895117 after retry RPC@1975516596) -> (continuation of Deferred@1256087222 after retry RPC@1074153476), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@508badd1 -> wakeup thread Time-limited test -> (continuation of Deferred@1405459676 after retry RPC@1580029520) -> (continuation of Deferred@812923913 after retry RPC@81752961) -> (continuation of Deferred@1219266070 after retry RPC@923838457) -> (continuation of Deferred@1920836104 after retry RPC@243679889) -> (continuation of Deferred@1225895117 after retry RPC@1975516596) -> (continuation of Deferred@1256087222 after retry RPC@1074153476))) after lookup
12:12:33.402 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:33.403 [DEBUG - kudu-nio-3] (Connection.java:757) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] Successfully connected to peer
12:12:33.422 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x7144e1e6, L:/127.0.0.1:41518 - R:/127.29.71.130:36197] using TOKEN/null
12:12:33.469 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.71.129:44009
12:12:33.469 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.470932 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/logs
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.129:44009
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.129
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42507
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.129
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:33.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.782413 31345 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:33.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.782871 31345 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:33.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.783406 31345 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.814524 31345 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:33.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.815317 31345 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.129
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.849828 31345 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.129:44009
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.129
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42507
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.129
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/logs
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:33.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.851022 31345 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:33.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.852536 31345 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:33.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.864540 31353 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.266947 31352 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 31345
12:12:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.324188 31352 kernel_stack_watchdog.cc:198] Thread 31345 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 399ms:
12:12:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
12:12:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
12:12:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
12:12:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
12:12:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.866391 31354 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:35.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.325450 31355 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1457 milliseconds
12:12:35.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.328779 31345 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.463s	user 0.002s	sys 0.003s
12:12:35.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.329118 31345 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.463s	user 0.003s	sys 0.003s
12:12:35.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.330267 31356 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:35.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.330297 31345 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:35.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.331540 31345 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.333999 31345 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.335403 31345 hybrid_clock.cc:648] HybridClock initialized: now 1783512755335311 us; error 87 us; skew 500 ppm
12:12:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.336426 31345 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.342077 31345 webserver.cc:533] Webserver started at http://127.29.71.129:42507/ using document root <none> and password file <none>
12:12:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.342939 31345 fs_manager.cc:362] Metadata directory not provided
12:12:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.343143 31345 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.350497 31345 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.001s	sys 0.003s
12:12:35.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.354583 31369 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:35.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.355525 31345 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.004s
12:12:35.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.355837 31345 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:35.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "836381137da64858b3e96d2b8f127242"
12:12:35.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:09 on dist-test-slave-tp5p"
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.357715 31345 fs_report.cc:389] FS layout report
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/data
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.401437 31345 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:35.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.402911 31345 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:35.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.403373 31345 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:35.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.405875 31345 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.411621 31376 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:12:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.419404 31345 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.419682 31345 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.001s	sys 0.000s
12:12:35.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.420038 31345 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:35.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.424525 31345 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:35.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.424695 31345 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
12:12:35.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.425109 31376 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap starting.
12:12:35.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.443169 30998 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a -> Peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Couldn't send request to peer 836381137da64858b3e96d2b8f127242. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
12:12:35.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.483151 31376 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Log is configured to *not* fsync() on all Append() calls
12:12:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.540799 31376 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:35.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.541937 31376 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap complete.
12:12:35.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.543659 31376 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Time spent bootstrapping tablet: real 0.119s	user 0.083s	sys 0.031s
12:12:35.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.561846 31376 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.564864 31376 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 836381137da64858b3e96d2b8f127242, State: Initialized, Role: FOLLOWER
12:12:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.566048 31376 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.574240 31376 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Time spent starting tablet: real 0.030s	user 0.027s	sys 0.004s
12:12:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.604619 31345 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.129:44009
12:12:35.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.604841 31486 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.129:44009 every 8 connection(s)
12:12:35.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.607066 31345 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:35.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.613515 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 31345
12:12:35.614 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:35.642 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:35.643 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.71.130:36197
12:12:35.644 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:12:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.645220 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 30975
12:12:35.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.648747 31487 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:35.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.649295 31487 heartbeater.cc:461] Registering TS with master...
12:12:35.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.650563 31487 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.655232 30182 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.655709 31488 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.656038 31488 heartbeater.cc:461] Registering TS with master...
12:12:35.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.656924 31488 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:35.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.660115 30103 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.665627 31489 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.665907 31489 heartbeater.cc:461] Registering TS with master...
12:12:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.666738 31489 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:35.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.668994 30028 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:35.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.670997 30028 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.129:35287
12:12:35.673 [DEBUG - kudu-nio-3] (Connection.java:239) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] handling channelInactive
12:12:35.674 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state READY due to: connection closed
12:12:35.675 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:35.676 [DEBUG - Time-limited test] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:35.677 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.677 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.677 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.700 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:35.700 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:35.701 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [25ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@764332615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@5fbae19a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@382600c0 -> wakeup thread Time-limited test)) after lookup
12:12:35.701 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:35.702 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.702 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.702 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.719 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:35.720 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:35.720 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [25ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [26ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [27ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [27ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [44ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@764332615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@5fbae19a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@382600c0 -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857))) after lookup
12:12:35.721 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:35.722 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.722 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.722 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.760 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:35.760 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:35.760 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=85), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [25ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [26ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [27ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [27ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [44ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [45ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [47ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [47ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [84ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@764332615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@5fbae19a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857) -> (continuation of Deferred@891196073 after retry RPC@1955058715), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@382600c0 -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857) -> (continuation of Deferred@891196073 after retry RPC@1955058715))) after lookup
12:12:35.761 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:35.762 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.762 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.762 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.859 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:35.860 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:35.861 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=186), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [25ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [26ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [27ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [27ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [44ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [45ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [47ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [47ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [84ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [86ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [87ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [87ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [181ms] refreshing cache from master, [181ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [184ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@764332615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@5fbae19a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857) -> (continuation of Deferred@891196073 after retry RPC@1955058715) -> (continuation of Deferred@123576631 after retry RPC@1295030987), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@382600c0 -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857) -> (continuation of Deferred@891196073 after retry RPC@1955058715) -> (continuation of Deferred@123576631 after retry RPC@1295030987))) after lookup
12:12:35.862 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:35.863 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.863 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.863 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.960 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:35.960 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:35.960 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=285), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [25ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [26ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [27ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [27ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [44ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [45ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [47ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [47ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [84ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [86ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [87ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [87ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [181ms] refreshing cache from master, [181ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [184ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [187ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [188ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [189ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [281ms] refreshing cache from master, [281ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [284ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@764332615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@5fbae19a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857) -> (continuation of Deferred@891196073 after retry RPC@1955058715) -> (continuation of Deferred@123576631 after retry RPC@1295030987) -> (continuation of Deferred@1107866184 after retry RPC@287776978), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@382600c0 -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857) -> (continuation of Deferred@891196073 after retry RPC@1955058715) -> (continuation of Deferred@123576631 after retry RPC@1295030987) -> (continuation of Deferred@1107866184 after retry RPC@287776978))) after lookup
12:12:35.961 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:35.962 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.962 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:35.962 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:36.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.675127 31489 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.814381 31498 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:12:36.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.814800 31498 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:36.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.816963 31498 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:36.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:36.821734 31370 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111)
12:12:36.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:36.825595 31370 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111)
12:12:36.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.826723 31247 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09" is_pre_election: true
12:12:36.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.828007 31370 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 836381137da64858b3e96d2b8f127242; no voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09
12:12:36.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.828609 31498 raft_consensus.cc:2749] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
12:12:36.859 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:36.860 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:36.860 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1184), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [25ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [26ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [27ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [27ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [44ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [45ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [47ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [47ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [84ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [86ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [87ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [87ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [181ms] refreshing cache from master, [181ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [184ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [187ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [188ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [189ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [281ms] refreshing cache from master, [281ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [284ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [286ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [287ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [287ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [1181ms] refreshing cache from master, [1181ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1184ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@764332615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@5fbae19a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857) -> (continuation of Deferred@891196073 after retry RPC@1955058715) -> (continuation of Deferred@123576631 after retry RPC@1295030987) -> (continuation of Deferred@1107866184 after retry RPC@287776978) -> (continuation of Deferred@653323607 after retry RPC@242837739), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@382600c0 -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857) -> (continuation of Deferred@891196073 after retry RPC@1955058715) -> (continuation of Deferred@123576631 after retry RPC@1295030987) -> (continuation of Deferred@1107866184 after retry RPC@287776978) -> (continuation of Deferred@653323607 after retry RPC@242837739))) after lookup
12:12:36.860 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:36.862 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:36.862 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:36.862 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:36.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.918751 31503 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader a40db63d1c9748e99885c3264331297a)
12:12:36.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.919137 31503 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:36.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.921504 31503 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:36.925511 31178 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111)
12:12:36.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:36.929507 31178 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111)
12:12:36.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.933351 31441 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "c7651a898f6142c793a606c219090c09" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "836381137da64858b3e96d2b8f127242" is_pre_election: true
12:12:36.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.933990 31441 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c7651a898f6142c793a606c219090c09 in term 5.
12:12:36.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.935155 31180 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, c7651a898f6142c793a606c219090c09; no voters: a40db63d1c9748e99885c3264331297a
12:12:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.935817 31503 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 6 FOLLOWER]: Leader pre-election won for term 7
12:12:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.936053 31503 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 6 FOLLOWER]: Starting leader election (detected failure of leader a40db63d1c9748e99885c3264331297a)
12:12:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.936262 31503 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 6 FOLLOWER]: Advancing to term 7
12:12:36.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.940478 31503 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:36.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.941866 31503 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 7 election: Requested vote from peers 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:36.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.942473 31441 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "c7651a898f6142c793a606c219090c09" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "836381137da64858b3e96d2b8f127242"
12:12:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.942879 31441 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 5 FOLLOWER]: Advancing to term 7
12:12:36.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:36.945079 31178 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111)
12:12:36.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.949173 31441 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c7651a898f6142c793a606c219090c09 in term 7.
12:12:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.949920 31180 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, c7651a898f6142c793a606c219090c09; no voters: a40db63d1c9748e99885c3264331297a
12:12:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.950426 31503 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 7 FOLLOWER]: Leader election won for term 7
12:12:36.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.951838 31503 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 7 LEADER]: Becoming Leader. State: Replica: c7651a898f6142c793a606c219090c09, State: Running, Role: LEADER
12:12:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.952843 31503 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:36.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:36.960500 30028 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 reported cstate change: term changed from 6 to 7, leader changed from a40db63d1c9748e99885c3264331297a (127.29.71.130) to c7651a898f6142c793a606c219090c09 (127.29.71.131). New cstate: current_term: 7 leader_uuid: "c7651a898f6142c793a606c219090c09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: UNKNOWN } } }
12:12:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.331972 31441 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 7 FOLLOWER]: Refusing update from remote peer c7651a898f6142c793a606c219090c09: Log matching property violated. Preceding OpId in replica: term: 5 index: 16. Preceding OpId from leader: term: 7 index: 21. (index mismatch)
12:12:37.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.333364 31503 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
12:12:37.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:37.362920 31178 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 -> Peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Couldn't send request to peer a40db63d1c9748e99885c3264331297a. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:37.760 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:37.761 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:37.761 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2086), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [25ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [26ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [27ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [27ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [44ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [45ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [47ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [47ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [84ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [86ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [87ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [87ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [181ms] refreshing cache from master, [181ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [184ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [187ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [188ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [189ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [281ms] refreshing cache from master, [281ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [284ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [286ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [287ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [287ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [1181ms] refreshing cache from master, [1181ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1184ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [1185ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [1187ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [1187ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2081ms] refreshing cache from master, [2081ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [2085ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@764332615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@5fbae19a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857) -> (continuation of Deferred@891196073 after retry RPC@1955058715) -> (continuation of Deferred@123576631 after retry RPC@1295030987) -> (continuation of Deferred@1107866184 after retry RPC@287776978) -> (continuation of Deferred@653323607 after retry RPC@242837739) -> (continuation of Deferred@2124071392 after retry RPC@996266216), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@382600c0 -> wakeup thread Time-limited test -> (continuation of Deferred@507124345 after retry RPC@123256857) -> (continuation of Deferred@891196073 after retry RPC@1955058715) -> (continuation of Deferred@123576631 after retry RPC@1295030987) -> (continuation of Deferred@1107866184 after retry RPC@287776978) -> (continuation of Deferred@653323607 after retry RPC@242837739) -> (continuation of Deferred@2124071392 after retry RPC@996266216))) after lookup
12:12:37.762 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:37.763 [DEBUG - kudu-nio-3] (Connection.java:757) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] Successfully connected to peer
12:12:37.783 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x5090dafc, L:/127.0.0.1:39908 - R:/127.29.71.131:38005] using TOKEN/null
12:12:37.835 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.71.130:36197
12:12:37.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:12:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.838507 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/logs
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.130:36197
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.130
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42689
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.130
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:38.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:38.134461 31530 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:38.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:38.134949 31530 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:38.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:38.135399 31530 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:38.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:38.167598 31530 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:38.168473 31530 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.130
12:12:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:38.203435 31530 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.130:36197
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.130
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42689
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.130
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/logs
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:38.204660 31530 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:38.206164 31530 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:38.218506 31536 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:38.219038 31537 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:39.312974 31538 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
12:12:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:39.314019 31539 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.314075 31530 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:39.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.317123 31530 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:39.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.319155 31530 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:39.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.320482 31530 hybrid_clock.cc:648] HybridClock initialized: now 1783512759320445 us; error 38 us; skew 500 ppm
12:12:39.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.321316 31530 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:39.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.328687 31530 webserver.cc:533] Webserver started at http://127.29.71.130:42689/ using document root <none> and password file <none>
12:12:39.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.329607 31530 fs_manager.cc:362] Metadata directory not provided
12:12:39.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.329814 31530 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.338148 31530 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.006s
12:12:39.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.343043 31553 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:39.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.344055 31530 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.344381 31530 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a40db63d1c9748e99885c3264331297a"
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:12 on dist-test-slave-tp5p"
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.346202 31530 fs_report.cc:389] FS layout report
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/data
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:39.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.399781 31530 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.401547 31530 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.401957 31530 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:39.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.404547 31530 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:39.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.410001 31562 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:12:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.418304 31530 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.418558 31530 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.000s	sys 0.002s
12:12:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.418778 31530 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.423305 31530 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.423493 31530 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.001s	sys 0.001s
12:12:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.423993 31562 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Bootstrap starting.
12:12:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.496070 31562 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Log is configured to *not* fsync() on all Append() calls
12:12:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.556278 31562 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:39.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.557240 31562 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Bootstrap complete.
12:12:39.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.558890 31562 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Time spent bootstrapping tablet: real 0.135s	user 0.088s	sys 0.040s
12:12:39.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.576226 31562 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:39.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.578874 31562 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: a40db63d1c9748e99885c3264331297a, State: Initialized, Role: FOLLOWER
12:12:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.579916 31562 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:39.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.583956 31562 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Time spent starting tablet: real 0.025s	user 0.025s	sys 0.000s
12:12:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.594725 31530 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.130:36197
12:12:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.595000 31670 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.130:36197 every 8 connection(s)
12:12:39.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.597313 31530 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:39.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.605583 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 31530
12:12:39.606 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:39.635 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:39.637 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.71.131:38005
12:12:39.637 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:39.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.637816 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 31158
12:12:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.663614 31672 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.664052 31672 heartbeater.cc:461] Registering TS with master...
12:12:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.665585 31672 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:39.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.669817 30103 ts_manager.cc:194] Re-registered known tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:39.671 [DEBUG - kudu-nio-3] (Connection.java:239) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] handling channelInactive
12:12:39.671 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state READY due to: connection closed
12:12:39.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:39.673 [DEBUG - Time-limited test] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:39.674 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.674327 31671 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:39.675 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.674644 31671 heartbeater.cc:461] Registering TS with master...
12:12:39.675 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.675529 31671 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:39.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.676692 31673 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:39.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.677057 31673 heartbeater.cc:461] Registering TS with master...
12:12:39.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.677909 31673 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:39.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.679226 30181 ts_manager.cc:194] Re-registered known tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:39.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.680626 30028 ts_manager.cc:194] Re-registered known tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:39.682 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:39.682 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:39.682 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@768395182(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@181fa46f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3331af36 -> wakeup thread Time-limited test)) after lookup
12:12:39.683 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:39.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.683292 30028 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.130:38511
12:12:39.684 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.684 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.684 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.699 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:39.700 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:39.700 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [9ms] sending RPC to server c7651a898f6142c793a606c219090c09, [11ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [11ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@768395182(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@181fa46f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@338945335 after retry RPC@709210264), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3331af36 -> wakeup thread Time-limited test -> (continuation of Deferred@338945335 after retry RPC@709210264))) after lookup
12:12:39.700 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:39.701 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.701 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.701 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.740 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:39.740 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:39.741 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [9ms] sending RPC to server c7651a898f6142c793a606c219090c09, [11ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [11ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [27ms] sending RPC to server c7651a898f6142c793a606c219090c09, [28ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [28ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [67ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@768395182(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@181fa46f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@338945335 after retry RPC@709210264) -> (continuation of Deferred@947752706 after retry RPC@1069230565), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3331af36 -> wakeup thread Time-limited test -> (continuation of Deferred@338945335 after retry RPC@709210264) -> (continuation of Deferred@947752706 after retry RPC@1069230565))) after lookup
12:12:39.741 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:39.742 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.743 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.743 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.799 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:39.800 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:39.800 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=127), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [9ms] sending RPC to server c7651a898f6142c793a606c219090c09, [11ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [11ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [27ms] sending RPC to server c7651a898f6142c793a606c219090c09, [28ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [28ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [67ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [68ms] sending RPC to server c7651a898f6142c793a606c219090c09, [70ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [70ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [123ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [126ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@768395182(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@181fa46f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@338945335 after retry RPC@709210264) -> (continuation of Deferred@947752706 after retry RPC@1069230565) -> (continuation of Deferred@2044449629 after retry RPC@524888450), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3331af36 -> wakeup thread Time-limited test -> (continuation of Deferred@338945335 after retry RPC@709210264) -> (continuation of Deferred@947752706 after retry RPC@1069230565) -> (continuation of Deferred@2044449629 after retry RPC@524888450))) after lookup
12:12:39.801 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:39.802 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.802 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:39.802 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:40.180 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:40.181 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:40.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=508), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [9ms] sending RPC to server c7651a898f6142c793a606c219090c09, [11ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [11ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [27ms] sending RPC to server c7651a898f6142c793a606c219090c09, [28ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [28ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [67ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [68ms] sending RPC to server c7651a898f6142c793a606c219090c09, [70ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [70ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [123ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [126ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [128ms] sending RPC to server c7651a898f6142c793a606c219090c09, [129ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [129ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [503ms] refreshing cache from master, [503ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [507ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@768395182(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@181fa46f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@338945335 after retry RPC@709210264) -> (continuation of Deferred@947752706 after retry RPC@1069230565) -> (continuation of Deferred@2044449629 after retry RPC@524888450) -> (continuation of Deferred@875023704 after retry RPC@986422167), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3331af36 -> wakeup thread Time-limited test -> (continuation of Deferred@338945335 after retry RPC@709210264) -> (continuation of Deferred@947752706 after retry RPC@1069230565) -> (continuation of Deferred@2044449629 after retry RPC@524888450) -> (continuation of Deferred@875023704 after retry RPC@986422167))) after lookup
12:12:40.182 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:40.182 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:40.182 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:40.183 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.686935 31673 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:40.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.814119 31683 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader c7651a898f6142c793a606c219090c09)
12:12:40.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.814502 31683 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:40.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.815948 31683 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:40.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:40.822436 31370 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111)
12:12:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.827250 31625 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "a40db63d1c9748e99885c3264331297a" is_pre_election: true
12:12:40.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.827971 31625 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 836381137da64858b3e96d2b8f127242 in term 6.
12:12:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.829178 31370 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, a40db63d1c9748e99885c3264331297a; no voters: c7651a898f6142c793a606c219090c09
12:12:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.829787 31683 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 7 FOLLOWER]: Leader pre-election won for term 8
12:12:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.830049 31683 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 7 FOLLOWER]: Starting leader election (detected failure of leader c7651a898f6142c793a606c219090c09)
12:12:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.830264 31683 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 7 FOLLOWER]: Advancing to term 8
12:12:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.834575 31683 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.836136 31683 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 8 election: Requested vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.837555 31625 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "a40db63d1c9748e99885c3264331297a"
12:12:40.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.838009 31625 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 6 FOLLOWER]: Advancing to term 8
12:12:40.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:40.839260 31370 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111)
12:12:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.844228 31625 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 836381137da64858b3e96d2b8f127242 in term 8.
12:12:40.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.845048 31370 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, a40db63d1c9748e99885c3264331297a; no voters: c7651a898f6142c793a606c219090c09
12:12:40.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.845679 31683 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 8 FOLLOWER]: Leader election won for term 8
12:12:40.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.846740 31683 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 8 LEADER]: Becoming Leader. State: Replica: 836381137da64858b3e96d2b8f127242, State: Running, Role: LEADER
12:12:40.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.847450 31683 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:40.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.857417 30028 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 reported cstate change: term changed from 7 to 8, leader changed from c7651a898f6142c793a606c219090c09 (127.29.71.131) to 836381137da64858b3e96d2b8f127242 (127.29.71.129). New cstate: current_term: 8 leader_uuid: "836381137da64858b3e96d2b8f127242" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: UNKNOWN } } }
12:12:41.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.266072 31625 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 8 FOLLOWER]: Refusing update from remote peer 836381137da64858b3e96d2b8f127242: Log matching property violated. Preceding OpId in replica: term: 6 index: 20. Preceding OpId from leader: term: 8 index: 25. (index mismatch)
12:12:41.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.267431 31683 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
12:12:41.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.291980 31370 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 -> Peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Couldn't send request to peer c7651a898f6142c793a606c219090c09. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:41.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.848007 31370 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111) [suppressed 3 similar messages]
12:12:42.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:42.181 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:42.182 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2508), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [8ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [9ms] sending RPC to server c7651a898f6142c793a606c219090c09, [11ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [11ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [26ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [27ms] sending RPC to server c7651a898f6142c793a606c219090c09, [28ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [28ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [67ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [68ms] sending RPC to server c7651a898f6142c793a606c219090c09, [70ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [70ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [123ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [126ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [128ms] sending RPC to server c7651a898f6142c793a606c219090c09, [129ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [129ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [503ms] refreshing cache from master, [503ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [507ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [508ms] sending RPC to server c7651a898f6142c793a606c219090c09, [510ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [510ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2503ms] refreshing cache from master, [2504ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [2507ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@768395182(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@181fa46f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@338945335 after retry RPC@709210264) -> (continuation of Deferred@947752706 after retry RPC@1069230565) -> (continuation of Deferred@2044449629 after retry RPC@524888450) -> (continuation of Deferred@875023704 after retry RPC@986422167) -> (continuation of Deferred@2144858856 after retry RPC@430360998), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3331af36 -> wakeup thread Time-limited test -> (continuation of Deferred@338945335 after retry RPC@709210264) -> (continuation of Deferred@947752706 after retry RPC@1069230565) -> (continuation of Deferred@2044449629 after retry RPC@524888450) -> (continuation of Deferred@875023704 after retry RPC@986422167) -> (continuation of Deferred@2144858856 after retry RPC@430360998))) after lookup
12:12:42.182 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:42.183 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] Successfully connected to peer
12:12:42.202 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x66694570, L:/127.0.0.1:41256 - R:/127.29.71.129:44009] using TOKEN/null
12:12:42.243 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.71.131:38005
12:12:42.243 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:42.245064 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/logs
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:42.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.131:38005
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.131
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33861
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.131
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:42.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:42.557993 31720 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:42.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:42.558454 31720 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:42.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:42.558907 31720 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:42.590373 31720 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:42.591184 31720 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.131
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:42.651830 31720 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.131:38005
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.131
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33861
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.131
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/logs
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:42.653123 31720 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:42.654680 31720 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:42.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:42.666429 31728 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:43.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:43.383279 30059 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3840 after lost signal to thread 29996
12:12:43.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:43.384220 30059 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8860 after lost signal to thread 30062
12:12:43.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:43.856122 31370 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 -> Peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Couldn't send request to peer c7651a898f6142c793a606c219090c09. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
12:12:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:42.666967 31729 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.177131 31720 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.511s	user 0.488s	sys 0.970s
12:12:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.068738 31727 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 31720
12:12:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.178671 31720 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.512s	user 0.488s	sys 0.970s
12:12:44.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.179270 31731 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.189543 31730 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1516 milliseconds
12:12:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.189589 31720 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:44.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.191118 31720 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:44.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.193598 31720 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:44.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.195003 31720 hybrid_clock.cc:648] HybridClock initialized: now 1783512764194962 us; error 46 us; skew 500 ppm
12:12:44.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.195765 31720 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:44.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.202504 31720 webserver.cc:533] Webserver started at http://127.29.71.131:33861/ using document root <none> and password file <none>
12:12:44.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.203337 31720 fs_manager.cc:362] Metadata directory not provided
12:12:44.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.203534 31720 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.211133 31720 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.003s
12:12:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.215762 31746 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.216949 31720 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.000s
12:12:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.217273 31720 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c7651a898f6142c793a606c219090c09"
12:12:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:13 on dist-test-slave-tp5p"
12:12:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.219132 31720 fs_report.cc:389] FS layout report
12:12:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/data
12:12:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.266078 31720 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:44.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.267546 31720 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:44.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.268007 31720 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.271070 31720 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:44.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.277349 31753 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:12:44.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.285166 31720 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:44.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.285382 31720 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
12:12:44.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.285696 31720 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:44.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.290182 31720 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:44.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.290418 31720 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
12:12:44.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.290782 31753 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Bootstrap starting.
12:12:44.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.372641 31753 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Log is configured to *not* fsync() on all Append() calls
12:12:44.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.476619 31753 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:44.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.477638 31753 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Bootstrap complete.
12:12:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.479391 31753 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Time spent bootstrapping tablet: real 0.189s	user 0.114s	sys 0.061s
12:12:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.501793 31753 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.504948 31753 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: c7651a898f6142c793a606c219090c09, State: Initialized, Role: FOLLOWER
12:12:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.506253 31753 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:44.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.511730 31753 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Time spent starting tablet: real 0.032s	user 0.033s	sys 0.000s
12:12:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.530241 31720 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.131:38005
12:12:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.530853 31863 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.131:38005 every 8 connection(s)
12:12:44.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.532718 31720 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:44.536 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:44.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.535341 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 31720
12:12:44.575 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:44.576 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.71.129:44009
12:12:44.577 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:44.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.577469 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 31345
12:12:44.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.588522 31864 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:44.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.594137 31864 heartbeater.cc:461] Registering TS with master...
12:12:44.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.595321 31864 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:44.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.591292 31865 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:44.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.598102 31865 heartbeater.cc:461] Registering TS with master...
12:12:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.598942 31865 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.602604 30181 ts_manager.cc:194] Re-registered known tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:44.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.605098 30103 ts_manager.cc:194] Re-registered known tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.613520 31866 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.613896 31866 heartbeater.cc:461] Registering TS with master...
12:12:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.614780 31866 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:44.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.618094 30028 ts_manager.cc:194] Re-registered known tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:44.624 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] handling channelInactive
12:12:44.624 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state READY due to: connection closed
12:12:44.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.624593 30028 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.131:43871
12:12:44.627 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:44.628 [DEBUG - Time-limited test] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:44.629 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.629 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.629 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.639 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:44.640 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:44.640 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@354963154(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3aec4e28 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b93c938 -> wakeup thread Time-limited test)) after lookup
12:12:44.640 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:44.641 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.641 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.641 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.660 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:44.660 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:44.661 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [13ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [13ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@354963154(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3aec4e28 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b93c938 -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297))) after lookup
12:12:44.661 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:44.663 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.663 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.663 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.680 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:44.680 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:44.680 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=52), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [13ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [13ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [33ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [35ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [35ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [52ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@354963154(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3aec4e28 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297) -> (continuation of Deferred@1792133390 after retry RPC@560572418), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b93c938 -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297) -> (continuation of Deferred@1792133390 after retry RPC@560572418))) after lookup
12:12:44.681 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:44.682 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.682 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.682 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.801 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:44.802 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:44.802 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=174), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [13ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [13ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [33ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [35ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [35ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [52ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [53ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [54ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [54ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [168ms] refreshing cache from master, [168ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [173ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@354963154(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3aec4e28 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297) -> (continuation of Deferred@1792133390 after retry RPC@560572418) -> (continuation of Deferred@1866130918 after retry RPC@67074491), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b93c938 -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297) -> (continuation of Deferred@1792133390 after retry RPC@560572418) -> (continuation of Deferred@1866130918 after retry RPC@67074491))) after lookup
12:12:44.803 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:44.805 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.805 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:44.805 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:45.200 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:45.201 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:45.201 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=573), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [13ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [13ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [33ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [35ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [35ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [52ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [53ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [54ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [54ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [168ms] refreshing cache from master, [168ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [173ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [175ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [177ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [177ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [568ms] refreshing cache from master, [568ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [572ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@354963154(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3aec4e28 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297) -> (continuation of Deferred@1792133390 after retry RPC@560572418) -> (continuation of Deferred@1866130918 after retry RPC@67074491) -> (continuation of Deferred@1391621804 after retry RPC@953898191), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b93c938 -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297) -> (continuation of Deferred@1792133390 after retry RPC@560572418) -> (continuation of Deferred@1866130918 after retry RPC@67074491) -> (continuation of Deferred@1391621804 after retry RPC@953898191))) after lookup
12:12:45.202 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:45.203 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:45.203 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:45.203 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:45.300 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:45.301 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:45.301 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=673), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [13ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [13ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [33ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [35ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [35ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [52ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [53ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [54ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [54ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [168ms] refreshing cache from master, [168ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [173ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [175ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [177ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [177ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [568ms] refreshing cache from master, [568ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [572ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [573ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [575ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [575ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [668ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [672ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@354963154(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3aec4e28 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297) -> (continuation of Deferred@1792133390 after retry RPC@560572418) -> (continuation of Deferred@1866130918 after retry RPC@67074491) -> (continuation of Deferred@1391621804 after retry RPC@953898191) -> (continuation of Deferred@1743486143 after retry RPC@747788864), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b93c938 -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297) -> (continuation of Deferred@1792133390 after retry RPC@560572418) -> (continuation of Deferred@1866130918 after retry RPC@67074491) -> (continuation of Deferred@1391621804 after retry RPC@953898191) -> (continuation of Deferred@1743486143 after retry RPC@747788864))) after lookup
12:12:45.302 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:45.303 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:45.303 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:45.303 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:45.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.615495 31875 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 836381137da64858b3e96d2b8f127242)
12:12:45.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.615867 31875 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.618221 31875 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:45.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:45.621346 31556 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:45.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:45.625662 31556 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:45.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.628464 31866 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:45.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.630378 31818 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "a40db63d1c9748e99885c3264331297a" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09" is_pre_election: true
12:12:45.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.631199 31818 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a40db63d1c9748e99885c3264331297a in term 7.
12:12:45.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.632709 31554 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09; no voters: 836381137da64858b3e96d2b8f127242
12:12:45.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.633538 31875 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 8 FOLLOWER]: Leader pre-election won for term 9
12:12:45.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.633841 31875 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 8 FOLLOWER]: Starting leader election (detected failure of leader 836381137da64858b3e96d2b8f127242)
12:12:45.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.634169 31875 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 8 FOLLOWER]: Advancing to term 9
12:12:45.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.638357 31875 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:45.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.639739 31875 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 9 election: Requested vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:45.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.640393 31818 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "a40db63d1c9748e99885c3264331297a" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09"
12:12:45.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.640798 31818 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 7 FOLLOWER]: Advancing to term 9
12:12:45.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:45.643577 31556 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:45.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.647269 31818 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a40db63d1c9748e99885c3264331297a in term 9.
12:12:45.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.648098 31554 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09; no voters: 836381137da64858b3e96d2b8f127242
12:12:45.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.648838 31875 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 9 FOLLOWER]: Leader election won for term 9
12:12:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.650631 31875 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 9 LEADER]: Becoming Leader. State: Replica: a40db63d1c9748e99885c3264331297a, State: Running, Role: LEADER
12:12:45.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.651753 31875 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:45.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.660465 30028 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a reported cstate change: term changed from 8 to 9, leader changed from 836381137da64858b3e96d2b8f127242 (127.29.71.129) to a40db63d1c9748e99885c3264331297a (127.29.71.130). New cstate: current_term: 9 leader_uuid: "a40db63d1c9748e99885c3264331297a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: HEALTHY } } }
12:12:46.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:46.187922 31818 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 9 FOLLOWER]: Refusing update from remote peer a40db63d1c9748e99885c3264331297a: Log matching property violated. Preceding OpId in replica: term: 7 index: 24. Preceding OpId from leader: term: 9 index: 29. (index mismatch)
12:12:46.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:46.189414 31893 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [LEADER]: Connected to new peer: Peer: permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
12:12:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:46.203670 31895 mvcc.cc:204] Tried to move back new op lower bound from 7305268274000486400 to 7305268268439281664. Current Snapshot: MvccSnapshot[applied={T|T < 7305268274000486400}]
12:12:46.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:46.224634 31556 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a -> Peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Couldn't send request to peer 836381137da64858b3e96d2b8f127242. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:48.000 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:48.000 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:48.001 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=3373), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [1ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [12ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [13ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [13ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [33ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [35ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [35ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [52ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [53ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [54ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [54ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [168ms] refreshing cache from master, [168ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [173ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [175ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [177ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [177ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [568ms] refreshing cache from master, [568ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [572ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [573ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [575ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [575ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [668ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [672ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [673ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [675ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [675ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [3368ms] refreshing cache from master, [3368ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [3372ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@354963154(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@3aec4e28 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297) -> (continuation of Deferred@1792133390 after retry RPC@560572418) -> (continuation of Deferred@1866130918 after retry RPC@67074491) -> (continuation of Deferred@1391621804 after retry RPC@953898191) -> (continuation of Deferred@1743486143 after retry RPC@747788864) -> (continuation of Deferred@910879787 after retry RPC@1345017077), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b93c938 -> wakeup thread Time-limited test -> (continuation of Deferred@1146526962 after retry RPC@29972297) -> (continuation of Deferred@1792133390 after retry RPC@560572418) -> (continuation of Deferred@1866130918 after retry RPC@67074491) -> (continuation of Deferred@1391621804 after retry RPC@953898191) -> (continuation of Deferred@1743486143 after retry RPC@747788864) -> (continuation of Deferred@910879787 after retry RPC@1345017077))) after lookup
12:12:48.002 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:48.003 [DEBUG - kudu-nio-2] (Connection.java:757) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] Successfully connected to peer
12:12:48.018 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x75c3f14b, L:/127.0.0.1:46258 - R:/127.29.71.130:36197] using TOKEN/null
12:12:48.059 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.71.129:44009
12:12:48.060 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.061357 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/logs
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.129:44009
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.129
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42507
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.129
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:48.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.369010 31916 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.369490 31916 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:48.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.369918 31916 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:48.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.401523 31916 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:48.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.402341 31916 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.129
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.437897 31916 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.129:44009
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.129
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42507
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.129
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/logs
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.439112 31916 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:48.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.441279 31916 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.456769 31926 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:48.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.783208 31556 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a -> Peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Couldn't send request to peer 836381137da64858b3e96d2b8f127242. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
12:12:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.858901 31925 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 31916
12:12:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.899060 31916 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.440s	user 0.497s	sys 0.769s
12:12:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.457795 31927 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.900379 31916 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.441s	user 0.498s	sys 0.770s
12:12:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.900841 31928 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1440 milliseconds
12:12:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.901433 31929 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.901800 31916 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:49.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.905126 31916 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.907109 31916 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.908449 31916 hybrid_clock.cc:648] HybridClock initialized: now 1783512769908406 us; error 36 us; skew 500 ppm
12:12:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.909310 31916 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:49.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.914964 31916 webserver.cc:533] Webserver started at http://127.29.71.129:42507/ using document root <none> and password file <none>
12:12:49.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.915798 31916 fs_manager.cc:362] Metadata directory not provided
12:12:49.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.916023 31916 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.923427 31916 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.003s	sys 0.003s
12:12:49.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.927590 31941 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:49.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.928534 31916 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
12:12:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.928853 31916 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "836381137da64858b3e96d2b8f127242"
12:12:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:09 on dist-test-slave-tp5p"
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.930688 31916 fs_report.cc:389] FS layout report
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/data
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:49.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.994693 31916 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:49.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.996369 31916 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.996932 31916 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:49.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.999621 31916 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.005466 31948 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:12:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.013306 31916 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.013553 31916 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.002s	sys 0.000s
12:12:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.013886 31916 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:50.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.018707 31916 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:50.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.018898 31916 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
12:12:50.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.019371 31948 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap starting.
12:12:50.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.082283 31948 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Log is configured to *not* fsync() on all Append() calls
12:12:50.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.184484 31948 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:50.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.185454 31948 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap complete.
12:12:50.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.185515 31916 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.129:44009
12:12:50.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.185638 32055 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.129:44009 every 8 connection(s)
12:12:50.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.187183 31948 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Time spent bootstrapping tablet: real 0.168s	user 0.116s	sys 0.048s
12:12:50.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.188416 31916 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:50.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.198470 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 31916
12:12:50.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:50.220 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:50.222 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.71.130:36197
12:12:50.223 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:12:50.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.223786 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 31530
12:12:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.244675 31948 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.250711 31948 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 836381137da64858b3e96d2b8f127242, State: Initialized, Role: FOLLOWER
12:12:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.254279 31948 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:50.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.268240 32058 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:50.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.268718 32058 heartbeater.cc:461] Registering TS with master...
12:12:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.270063 32058 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.270485 31948 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Time spent starting tablet: real 0.083s	user 0.029s	sys 0.007s
12:12:50.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.272418 32057 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:50.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.272754 32057 heartbeater.cc:461] Registering TS with master...
12:12:50.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.273675 32057 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:50.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.274585 30026 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:50.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.275552 32056 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.275892 32056 heartbeater.cc:461] Registering TS with master...
12:12:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.276734 32056 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.277338 30102 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.277726 30026 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.129:49589
12:12:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.280229 30181 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.281548 32058 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:50.285 [DEBUG - kudu-nio-2] (Connection.java:239) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] handling channelInactive
12:12:50.285 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state READY due to: connection closed
12:12:50.287 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:50.288 [DEBUG - Time-limited test] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:50.289 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.289 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.289 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.299 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:50.299 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:50.300 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@421447668(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@1318b4de -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74721ced -> wakeup thread Time-limited test)) after lookup
12:12:50.300 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:50.301 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.301 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.301 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.320 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:50.320 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:50.320 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [13ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [14ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [14ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@421447668(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@1318b4de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@392233901 after retry RPC@1930485976), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74721ced -> wakeup thread Time-limited test -> (continuation of Deferred@392233901 after retry RPC@1930485976))) after lookup
12:12:50.321 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:50.322 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.322 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.322 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.359 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:50.359 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:50.360 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [13ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [14ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [14ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [34ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [35ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [35ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [72ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@421447668(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@1318b4de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@392233901 after retry RPC@1930485976) -> (continuation of Deferred@1908029293 after retry RPC@98045858), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74721ced -> wakeup thread Time-limited test -> (continuation of Deferred@392233901 after retry RPC@1930485976) -> (continuation of Deferred@1908029293 after retry RPC@98045858))) after lookup
12:12:50.360 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:50.361 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.361 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.361 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.440 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:50.440 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:50.441 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=153), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [13ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [14ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [14ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [34ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [35ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [35ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [72ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [73ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [74ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [74ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [153ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@421447668(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@1318b4de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@392233901 after retry RPC@1930485976) -> (continuation of Deferred@1908029293 after retry RPC@98045858) -> (continuation of Deferred@1432356912 after retry RPC@268478497), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74721ced -> wakeup thread Time-limited test -> (continuation of Deferred@392233901 after retry RPC@1930485976) -> (continuation of Deferred@1908029293 after retry RPC@98045858) -> (continuation of Deferred@1432356912 after retry RPC@268478497))) after lookup
12:12:50.441 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:50.443 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.443 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.443 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.719 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:50.720 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:50.720 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=432), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [13ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [14ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [14ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [34ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [35ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [35ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [72ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [73ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [74ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [74ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [153ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [154ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [156ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [156ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [429ms] refreshing cache from master, [429ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [432ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@421447668(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@1318b4de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@392233901 after retry RPC@1930485976) -> (continuation of Deferred@1908029293 after retry RPC@98045858) -> (continuation of Deferred@1432356912 after retry RPC@268478497) -> (continuation of Deferred@1254230220 after retry RPC@1603727381), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74721ced -> wakeup thread Time-limited test -> (continuation of Deferred@392233901 after retry RPC@1930485976) -> (continuation of Deferred@1908029293 after retry RPC@98045858) -> (continuation of Deferred@1432356912 after retry RPC@268478497) -> (continuation of Deferred@1254230220 after retry RPC@1603727381))) after lookup
12:12:50.721 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:50.722 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.722 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state CONNECTING due to: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:50.722 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a40db63d1c9748e99885c3264331297a(127.29.71.130:36197) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197
12:12:51.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.687172 32070 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader a40db63d1c9748e99885c3264331297a)
12:12:51.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.687589 32070 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:51.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.689996 32070 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:51.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:51.693359 31747 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111)
12:12:51.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:51.697502 31747 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111)
12:12:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.701967 32011 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "c7651a898f6142c793a606c219090c09" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "836381137da64858b3e96d2b8f127242" is_pre_election: true
12:12:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.702595 32011 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c7651a898f6142c793a606c219090c09 in term 8.
12:12:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.703764 31749 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, c7651a898f6142c793a606c219090c09; no voters: a40db63d1c9748e99885c3264331297a
12:12:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.704367 32070 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 9 FOLLOWER]: Leader pre-election won for term 10
12:12:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.704608 32070 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 9 FOLLOWER]: Starting leader election (detected failure of leader a40db63d1c9748e99885c3264331297a)
12:12:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.704816 32070 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 9 FOLLOWER]: Advancing to term 10
12:12:51.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.709419 32070 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:51.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.710744 32070 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 10 election: Requested vote from peers 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:51.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.711509 32011 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "c7651a898f6142c793a606c219090c09" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "836381137da64858b3e96d2b8f127242"
12:12:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.711932 32011 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 8 FOLLOWER]: Advancing to term 10
12:12:51.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:51.713872 31747 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111)
12:12:51.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.718137 32011 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c7651a898f6142c793a606c219090c09 in term 10.
12:12:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.718899 31749 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, c7651a898f6142c793a606c219090c09; no voters: a40db63d1c9748e99885c3264331297a
12:12:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.719406 32070 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 10 FOLLOWER]: Leader election won for term 10
12:12:51.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.720795 32070 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 10 LEADER]: Becoming Leader. State: Replica: c7651a898f6142c793a606c219090c09, State: Running, Role: LEADER
12:12:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.721617 32070 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.730353 30026 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 reported cstate change: term changed from 9 to 10, leader changed from a40db63d1c9748e99885c3264331297a (127.29.71.130) to c7651a898f6142c793a606c219090c09 (127.29.71.131). New cstate: current_term: 10 leader_uuid: "c7651a898f6142c793a606c219090c09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: UNKNOWN } } }
12:12:51.820 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:51.821 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:51.821 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1533), Traces: [0ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [2ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [2ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [12ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [13ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [14ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [14ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [34ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [35ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [35ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [72ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [73ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [74ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [74ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [153ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [154ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [156ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [156ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [429ms] refreshing cache from master, [429ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [432ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [433ms] sending RPC to server a40db63d1c9748e99885c3264331297a, [435ms] received response from server a40db63d1c9748e99885c3264331297a: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [435ms] delaying RPC due to: Network error: Failed to connect to peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197): Connection refused: /127.29.71.130:36197, [1529ms] refreshing cache from master, [1529ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1533ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@421447668(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@1318b4de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@392233901 after retry RPC@1930485976) -> (continuation of Deferred@1908029293 after retry RPC@98045858) -> (continuation of Deferred@1432356912 after retry RPC@268478497) -> (continuation of Deferred@1254230220 after retry RPC@1603727381) -> (continuation of Deferred@646546670 after retry RPC@610166352), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@74721ced -> wakeup thread Time-limited test -> (continuation of Deferred@392233901 after retry RPC@1930485976) -> (continuation of Deferred@1908029293 after retry RPC@98045858) -> (continuation of Deferred@1432356912 after retry RPC@268478497) -> (continuation of Deferred@1254230220 after retry RPC@1603727381) -> (continuation of Deferred@646546670 after retry RPC@610166352))) after lookup
12:12:51.822 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:51.822 [DEBUG - kudu-nio-1] (Connection.java:757) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] Successfully connected to peer
12:12:51.840 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xcb6f7aba, L:/127.0.0.1:59796 - R:/127.29.71.131:38005] using TOKEN/null
12:12:51.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.849835 32011 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 10 FOLLOWER]: Refusing update from remote peer c7651a898f6142c793a606c219090c09: Log matching property violated. Preceding OpId in replica: term: 8 index: 28. Preceding OpId from leader: term: 10 index: 34. (index mismatch)
12:12:51.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.851312 32075 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
12:12:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:51.851877 31747 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 -> Peer a40db63d1c9748e99885c3264331297a (127.29.71.130:36197): Couldn't send request to peer a40db63d1c9748e99885c3264331297a. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.130:36197: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:51.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.864804 32091 mvcc.cc:204] Tried to move back new op lower bound from 7305268297820712960 to 7305268288117547008. Current Snapshot: MvccSnapshot[applied={T|T < 7305268297820712960}]
12:12:51.908 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.71.130:36197
12:12:51.908 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.909986 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/logs
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.130:36197
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.130
12:12:51.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:51.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42689
12:12:51.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.130
12:12:51.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:51.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:51.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:51.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:52.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:52.196029 32098 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:52.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:52.196467 32098 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:52.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:52.196952 32098 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:52.230230 32098 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:52.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:52.231115 32098 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.130
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:52.267880 32098 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.130:36197
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.130
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42689
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.130
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/logs
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:52.269212 32098 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:52.270817 32098 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:52.282620 32104 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:53.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:52.283046 32105 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:53.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:53.672240 32106 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1386 milliseconds
12:12:53.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:53.672824 32107 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:53.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:53.676069 32098 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.394s	user 0.501s	sys 0.867s
12:12:53.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:53.676553 32098 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.394s	user 0.503s	sys 0.869s
12:12:53.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.676848 32098 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.678333 32098 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:53.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.680984 32098 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:53.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.682389 32098 hybrid_clock.cc:648] HybridClock initialized: now 1783512773682277 us; error 109 us; skew 500 ppm
12:12:53.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.683233 32098 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:53.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.689944 32098 webserver.cc:533] Webserver started at http://127.29.71.130:42689/ using document root <none> and password file <none>
12:12:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.690861 32098 fs_manager.cc:362] Metadata directory not provided
12:12:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.691094 32098 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:53.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.698732 32098 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.003s
12:12:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.703586 32118 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.704741 32098 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.705082 32098 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a40db63d1c9748e99885c3264331297a"
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:12 on dist-test-slave-tp5p"
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.707167 32098 fs_report.cc:389] FS layout report
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/wal
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/data
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.784572 32098 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.786118 32098 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.786602 32098 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.789580 32098 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:53.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.801731 32125 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:12:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.810009 32098 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.810293 32098 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
12:12:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.810583 32098 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.817104 32098 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.817344 32098 ts_tablet_manager.cc:595] Time spent register tablets: real 0.007s	user 0.007s	sys 0.000s
12:12:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.817715 32125 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Bootstrap starting.
12:12:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.882550 32125 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Log is configured to *not* fsync() on all Append() calls
12:12:53.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.986018 32098 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.130:36197
12:12:53.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.986106 32232 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.130:36197 every 8 connection(s)
12:12:53.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.989667 32098 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-1/data/info.pb
12:12:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.994403 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 32098
12:12:53.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.996341 32125 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.997402 32125 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Bootstrap complete.
12:12:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:53.999166 32125 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Time spent bootstrapping tablet: real 0.182s	user 0.145s	sys 0.030s
12:12:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.023340 32125 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.029259 32125 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: a40db63d1c9748e99885c3264331297a, State: Initialized, Role: FOLLOWER
12:12:54.031 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:54.032 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.71.131:38005
12:12:54.032 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:54.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.031740 32125 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:54.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.032655 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 31720
12:12:54.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.043598 32125 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a: Time spent starting tablet: real 0.044s	user 0.022s	sys 0.012s
12:12:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.049403 32233 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:54.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.049851 32233 heartbeater.cc:461] Registering TS with master...
12:12:54.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.051024 32233 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:54.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.052073 32234 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:54.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.052402 32234 heartbeater.cc:461] Registering TS with master...
12:12:54.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.053274 32234 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:54.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.056056 30181 ts_manager.cc:194] Re-registered known tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:54.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.056728 30102 ts_manager.cc:194] Re-registered known tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.058411 32235 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.058736 32235 heartbeater.cc:461] Registering TS with master...
12:12:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.059587 32235 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:54.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.062489 30026 ts_manager.cc:194] Re-registered known tserver with Master: a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:54.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.064592 30026 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.130:36359
12:12:54.065 [DEBUG - kudu-nio-1] (Connection.java:239) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] handling channelInactive
12:12:54.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state READY due to: connection closed
12:12:54.067 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:54.067 [DEBUG - Time-limited test] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:54.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:54.067745 32235 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:54.068 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.068 [DEBUG - kudu-nio-2] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.069 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.080 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:54.080 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:54.080 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1420469981(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@31136fe -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12a34d56 -> wakeup thread Time-limited test)) after lookup
12:12:54.081 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:54.082 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.082 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.082 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.099 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:54.099 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:54.099 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [14ms] sending RPC to server c7651a898f6142c793a606c219090c09, [15ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [15ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1420469981(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@31136fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12a34d56 -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325))) after lookup
12:12:54.100 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:54.101 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.101 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.101 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.140 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:54.140 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:54.141 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [14ms] sending RPC to server c7651a898f6142c793a606c219090c09, [15ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [15ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [33ms] sending RPC to server c7651a898f6142c793a606c219090c09, [34ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [34ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1420469981(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@31136fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325) -> (continuation of Deferred@205967539 after retry RPC@908373760), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12a34d56 -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325) -> (continuation of Deferred@205967539 after retry RPC@908373760))) after lookup
12:12:54.141 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:54.142 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.142 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.142 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.240 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:54.240 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:54.241 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=173), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [14ms] sending RPC to server c7651a898f6142c793a606c219090c09, [15ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [15ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [33ms] sending RPC to server c7651a898f6142c793a606c219090c09, [34ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [34ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [74ms] sending RPC to server c7651a898f6142c793a606c219090c09, [75ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [75ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [173ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1420469981(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@31136fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325) -> (continuation of Deferred@205967539 after retry RPC@908373760) -> (continuation of Deferred@338269495 after retry RPC@132543517), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12a34d56 -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325) -> (continuation of Deferred@205967539 after retry RPC@908373760) -> (continuation of Deferred@338269495 after retry RPC@132543517))) after lookup
12:12:54.241 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:54.242 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.242 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.242 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.440 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:54.440 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:54.441 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=373), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [14ms] sending RPC to server c7651a898f6142c793a606c219090c09, [15ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [15ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [33ms] sending RPC to server c7651a898f6142c793a606c219090c09, [34ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [34ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [74ms] sending RPC to server c7651a898f6142c793a606c219090c09, [75ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [75ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [173ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [174ms] sending RPC to server c7651a898f6142c793a606c219090c09, [175ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [175ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [369ms] refreshing cache from master, [369ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [373ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1420469981(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@31136fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325) -> (continuation of Deferred@205967539 after retry RPC@908373760) -> (continuation of Deferred@338269495 after retry RPC@132543517) -> (continuation of Deferred@549889069 after retry RPC@347372212), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12a34d56 -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325) -> (continuation of Deferred@205967539 after retry RPC@908373760) -> (continuation of Deferred@338269495 after retry RPC@132543517) -> (continuation of Deferred@549889069 after retry RPC@347372212))) after lookup
12:12:54.441 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:54.443 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.443 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:54.443 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:55.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.107196 32245 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader c7651a898f6142c793a606c219090c09)
12:12:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.107746 32245 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.110720 32245 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:55.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.118752 31942 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111)
12:12:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.124380 31942 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111)
12:12:55.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.133476 32188 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "a40db63d1c9748e99885c3264331297a" is_pre_election: true
12:12:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.134093 32188 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 836381137da64858b3e96d2b8f127242 in term 9.
12:12:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.135185 31942 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, a40db63d1c9748e99885c3264331297a; no voters: c7651a898f6142c793a606c219090c09
12:12:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.135790 32245 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 10 FOLLOWER]: Leader pre-election won for term 11
12:12:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.136092 32245 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 10 FOLLOWER]: Starting leader election (detected failure of leader c7651a898f6142c793a606c219090c09)
12:12:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.136317 32245 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 10 FOLLOWER]: Advancing to term 11
12:12:55.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.140475 32245 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.141930 32245 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 11 election: Requested vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), a40db63d1c9748e99885c3264331297a (127.29.71.130:36197)
12:12:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.143342 32188 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "836381137da64858b3e96d2b8f127242" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "a40db63d1c9748e99885c3264331297a"
12:12:55.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.143741 32188 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 9 FOLLOWER]: Advancing to term 11
12:12:55.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.145328 31942 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111)
12:12:55.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.150043 32188 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 836381137da64858b3e96d2b8f127242 in term 11.
12:12:55.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.150816 31942 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 836381137da64858b3e96d2b8f127242, a40db63d1c9748e99885c3264331297a; no voters: c7651a898f6142c793a606c219090c09
12:12:55.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.151376 32245 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 11 FOLLOWER]: Leader election won for term 11
12:12:55.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.152782 32245 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 11 LEADER]: Becoming Leader. State: Replica: 836381137da64858b3e96d2b8f127242, State: Running, Role: LEADER
12:12:55.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.153607 32245 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:55.160 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:55.160 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)[L]]}]
12:12:55.161 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1093), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [14ms] sending RPC to server c7651a898f6142c793a606c219090c09, [15ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [15ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [33ms] sending RPC to server c7651a898f6142c793a606c219090c09, [34ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [34ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [74ms] sending RPC to server c7651a898f6142c793a606c219090c09, [75ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [75ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [173ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [174ms] sending RPC to server c7651a898f6142c793a606c219090c09, [175ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [175ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [369ms] refreshing cache from master, [369ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [373ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [374ms] sending RPC to server c7651a898f6142c793a606c219090c09, [376ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [376ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [1089ms] refreshing cache from master, [1089ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1093ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1420469981(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@31136fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325) -> (continuation of Deferred@205967539 after retry RPC@908373760) -> (continuation of Deferred@338269495 after retry RPC@132543517) -> (continuation of Deferred@549889069 after retry RPC@347372212) -> (continuation of Deferred@515163860 after retry RPC@694320748), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12a34d56 -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325) -> (continuation of Deferred@205967539 after retry RPC@908373760) -> (continuation of Deferred@338269495 after retry RPC@132543517) -> (continuation of Deferred@549889069 after retry RPC@347372212) -> (continuation of Deferred@515163860 after retry RPC@694320748))) after lookup
12:12:55.162 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] connecting to peer
12:12:55.163 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:55.164 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] cleaning up while in state CONNECTING due to: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:55.164 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c7651a898f6142c793a606c219090c09(127.29.71.131:38005) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005
12:12:55.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.165205 30026 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 reported cstate change: term changed from 10 to 11, leader changed from c7651a898f6142c793a606c219090c09 (127.29.71.131) to 836381137da64858b3e96d2b8f127242 (127.29.71.129). New cstate: current_term: 11 leader_uuid: "836381137da64858b3e96d2b8f127242" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: UNKNOWN } } }
12:12:55.299 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:55.300 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:55.300 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1233), Traces: [0ms] sending RPC to server c7651a898f6142c793a606c219090c09, [2ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [2ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [14ms] sending RPC to server c7651a898f6142c793a606c219090c09, [15ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [15ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [33ms] sending RPC to server c7651a898f6142c793a606c219090c09, [34ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [34ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [73ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [74ms] sending RPC to server c7651a898f6142c793a606c219090c09, [75ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [75ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [173ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [174ms] sending RPC to server c7651a898f6142c793a606c219090c09, [175ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [175ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [369ms] refreshing cache from master, [369ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [373ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [374ms] sending RPC to server c7651a898f6142c793a606c219090c09, [376ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [376ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [1089ms] refreshing cache from master, [1089ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1093ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [1095ms] sending RPC to server c7651a898f6142c793a606c219090c09, [1097ms] received response from server c7651a898f6142c793a606c219090c09: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [1097ms] delaying RPC due to: Network error: Failed to connect to peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005): Connection refused: /127.29.71.131:38005, [1229ms] refreshing cache from master, [1229ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1232ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1420469981(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@31136fe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325) -> (continuation of Deferred@205967539 after retry RPC@908373760) -> (continuation of Deferred@338269495 after retry RPC@132543517) -> (continuation of Deferred@549889069 after retry RPC@347372212) -> (continuation of Deferred@515163860 after retry RPC@694320748) -> (continuation of Deferred@216725857 after retry RPC@1726871549), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12a34d56 -> wakeup thread Time-limited test -> (continuation of Deferred@1349345339 after retry RPC@1487687325) -> (continuation of Deferred@205967539 after retry RPC@908373760) -> (continuation of Deferred@338269495 after retry RPC@132543517) -> (continuation of Deferred@549889069 after retry RPC@347372212) -> (continuation of Deferred@515163860 after retry RPC@694320748) -> (continuation of Deferred@216725857 after retry RPC@1726871549))) after lookup
12:12:55.301 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:55.301 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] Successfully connected to peer
12:12:55.318 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2774450f, L:/127.0.0.1:38584 - R:/127.29.71.129:44009] using TOKEN/null
12:12:55.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.329780 32188 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 11 FOLLOWER]: Refusing update from remote peer 836381137da64858b3e96d2b8f127242: Log matching property violated. Preceding OpId in replica: term: 9 index: 32. Preceding OpId from leader: term: 11 index: 38. (index mismatch)
12:12:55.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.331691 32250 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.001s
12:12:55.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.332293 31942 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 -> Peer c7651a898f6142c793a606c219090c09 (127.29.71.131:38005): Couldn't send request to peer c7651a898f6142c793a606c219090c09. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.131:38005: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.368990 32267 mvcc.cc:204] Tried to move back new op lower bound from 7305268327736987648 to 7305268327035891712. Current Snapshot: MvccSnapshot[applied={T|T < 7305268327736987648 or (T in {7305268327736987648})}]
12:12:55.390 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.71.131:38005
12:12:55.390 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.391680 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/logs
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.131:38005
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.131
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33861
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.131
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:55.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.680343 32274 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:55.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.680788 32274 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:55.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.681264 32274 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:55.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.713452 32274 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:55.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.714238 32274 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.131
12:12:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.749562 32274 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.131:38005
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.131
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33861
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.131
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/logs
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.750790 32274 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:55.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.752923 32274 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.766131 32280 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:56.114732 30059 debug-util.cc:398] Leaking SignalData structure 0x7b08000b60c0 after lost signal to thread 29996
12:12:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:56.115465 30059 debug-util.cc:398] Leaking SignalData structure 0x7b08000aa560 after lost signal to thread 30062
12:12:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:57.169111 32279 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 32274
12:12:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.767894 32281 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:57.182782 32274 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.415s	user 0.600s	sys 0.808s
12:12:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:57.183108 32274 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.416s	user 0.600s	sys 0.808s
12:12:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:57.184923 32283 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:57.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:57.187769 32282 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1419 milliseconds
12:12:57.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.187783 32274 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:12:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.189070 32274 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:57.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.191191 32274 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:57.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.192529 32274 hybrid_clock.cc:648] HybridClock initialized: now 1783512777192487 us; error 44 us; skew 500 ppm
12:12:57.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.193375 32274 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:12:57.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.199256 32274 webserver.cc:533] Webserver started at http://127.29.71.131:33861/ using document root <none> and password file <none>
12:12:57.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.200217 32274 fs_manager.cc:362] Metadata directory not provided
12:12:57.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.200472 32274 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:57.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.208425 32274 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.001s
12:12:57.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.213398 32290 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.214395 32274 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.004s
12:12:57.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.214720 32274 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:57.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c7651a898f6142c793a606c219090c09"
12:12:57.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:13 on dist-test-slave-tp5p"
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.216650 32274 fs_report.cc:389] FS layout report
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/wal
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/data
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:57.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.301745 32274 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:57.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.303383 32274 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:57.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.303898 32274 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.306561 32274 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:57.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.312487 32297 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:12:57.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.320163 32274 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:57.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.320406 32274 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.002s	sys 0.000s
12:12:57.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.320713 32274 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:57.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.325264 32274 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:57.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.325443 32274 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
12:12:57.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.325752 32297 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Bootstrap starting.
12:12:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.398808 32297 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Log is configured to *not* fsync() on all Append() calls
12:12:57.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.496740 32274 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.131:38005
12:12:57.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.496809 32404 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.131:38005 every 8 connection(s)
12:12:57.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.500237 32274 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-2/data/info.pb
12:12:57.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.508205 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 32274
12:12:57.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:57.544 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:57.545 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.71.129:44009
12:12:57.545 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.546204 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 31916
12:12:57.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.556313 32297 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:57.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.557284 32297 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Bootstrap complete.
12:12:57.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.559362 32297 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Time spent bootstrapping tablet: real 0.234s	user 0.153s	sys 0.045s
12:12:57.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.565481 32405 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:12:57.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.566193 32405 heartbeater.cc:461] Registering TS with master...
12:12:57.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.567431 32405 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:12:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.570734 32406 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:12:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.571046 32406 heartbeater.cc:461] Registering TS with master...
12:12:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.571929 32406 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:12:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.572376 30181 ts_manager.cc:194] Re-registered known tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:57.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.575235 30102 ts_manager.cc:194] Re-registered known tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.577580 32407 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:12:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.577876 32407 heartbeater.cc:461] Registering TS with master...
12:12:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.578600 32407 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:12:57.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.577348 32297 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:57.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.580339 32297 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: c7651a898f6142c793a606c219090c09, State: Initialized, Role: FOLLOWER
12:12:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.581228 32297 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:57.586 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] handling channelInactive
12:12:57.587 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state READY due to: connection closed
12:12:57.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.586325 32297 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09: Time spent starting tablet: real 0.027s	user 0.019s	sys 0.005s
12:12:57.588 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:57.588 [DEBUG - Time-limited test] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:57.590 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.590 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.590 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.591408 30028 ts_manager.cc:194] Re-registered known tserver with Master: c7651a898f6142c793a606c219090c09 (127.29.71.131:38005)
12:12:57.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.594193 30028 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.131:35399
12:12:57.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.597275 32407 heartbeater.cc:499] Master 127.29.71.190:46473 was elected leader, sending a full tablet report...
12:12:57.600 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:57.600 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:57.600 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1526783238(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@525b8b84 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2c4d8cc1 -> wakeup thread Time-limited test)) after lookup
12:12:57.601 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:57.602 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.602 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.602 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.619 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:57.619 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:57.619 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [13ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [14ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [14ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1526783238(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@525b8b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@800548966 after retry RPC@1868350165), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2c4d8cc1 -> wakeup thread Time-limited test -> (continuation of Deferred@800548966 after retry RPC@1868350165))) after lookup
12:12:57.620 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:57.621 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.621 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.621 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.660 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:57.660 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:57.660 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [13ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [14ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [14ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [31ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [33ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [33ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [72ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1526783238(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@525b8b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@800548966 after retry RPC@1868350165) -> (continuation of Deferred@1641630068 after retry RPC@69366111), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2c4d8cc1 -> wakeup thread Time-limited test -> (continuation of Deferred@800548966 after retry RPC@1868350165) -> (continuation of Deferred@1641630068 after retry RPC@69366111))) after lookup
12:12:57.661 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:57.662 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.662 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.662 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.780 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:57.780 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:57.781 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=192), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [13ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [14ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [14ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [31ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [33ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [33ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [72ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [73ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [74ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [74ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [188ms] refreshing cache from master, [188ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [191ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1526783238(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@525b8b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@800548966 after retry RPC@1868350165) -> (continuation of Deferred@1641630068 after retry RPC@69366111) -> (continuation of Deferred@1061367752 after retry RPC@679217654), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2c4d8cc1 -> wakeup thread Time-limited test -> (continuation of Deferred@800548966 after retry RPC@1868350165) -> (continuation of Deferred@1641630068 after retry RPC@69366111) -> (continuation of Deferred@1061367752 after retry RPC@679217654))) after lookup
12:12:57.781 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:57.782 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.782 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:57.782 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:58.020 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:58.020 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009)[L],a40db63d1c9748e99885c3264331297a(127.29.71.130:36197),c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:58.020 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=432), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [13ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [14ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [14ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [31ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [33ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [33ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [72ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [73ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [74ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [74ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [188ms] refreshing cache from master, [188ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [191ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [193ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [194ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [194ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [428ms] refreshing cache from master, [428ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [432ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1526783238(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@525b8b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@800548966 after retry RPC@1868350165) -> (continuation of Deferred@1641630068 after retry RPC@69366111) -> (continuation of Deferred@1061367752 after retry RPC@679217654) -> (continuation of Deferred@580720617 after retry RPC@1793493493), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2c4d8cc1 -> wakeup thread Time-limited test -> (continuation of Deferred@800548966 after retry RPC@1868350165) -> (continuation of Deferred@1641630068 after retry RPC@69366111) -> (continuation of Deferred@1061367752 after retry RPC@679217654) -> (continuation of Deferred@580720617 after retry RPC@1793493493))) after lookup
12:12:58.021 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] connecting to peer
12:12:58.022 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:58.022 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] cleaning up while in state CONNECTING due to: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:58.022 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 836381137da64858b3e96d2b8f127242(127.29.71.129:44009) for tablet 6a1f398f09354ae5af27df2891707f0c: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009
12:12:58.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.587030 32417 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 11 FOLLOWER]: Starting pre-election (detected failure of leader 836381137da64858b3e96d2b8f127242)
12:12:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.587467 32417 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 11 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.589665 32417 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 12 pre-election: Requested pre-vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:58.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.592744 32121 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.596112 32121 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 12 pre-election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.602272 32360 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "a40db63d1c9748e99885c3264331297a" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09" is_pre_election: true
12:12:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.602981 32360 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a40db63d1c9748e99885c3264331297a in term 10.
12:12:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.604365 32119 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 12 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09; no voters: 836381137da64858b3e96d2b8f127242
12:12:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.605192 32417 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 11 FOLLOWER]: Leader pre-election won for term 12
12:12:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.605477 32417 raft_consensus.cc:493] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 11 FOLLOWER]: Starting leader election (detected failure of leader 836381137da64858b3e96d2b8f127242)
12:12:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.605718 32417 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 11 FOLLOWER]: Advancing to term 12
12:12:58.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.610150 32417 raft_consensus.cc:515] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 12 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.611653 32417 leader_election.cc:290] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 12 election: Requested vote from peers c7651a898f6142c793a606c219090c09 (127.29.71.131:38005), 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:12:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.612383 32360 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "6a1f398f09354ae5af27df2891707f0c" candidate_uuid: "a40db63d1c9748e99885c3264331297a" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "c7651a898f6142c793a606c219090c09"
12:12:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.612937 32360 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 10 FOLLOWER]: Advancing to term 12
12:12:58.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.615226 32121 leader_election.cc:336] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 12 election: RPC error from VoteRequest() call to peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111)
12:12:58.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.619441 32360 raft_consensus.cc:2468] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 12 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a40db63d1c9748e99885c3264331297a in term 12.
12:12:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.620203 32119 leader_election.cc:304] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [CANDIDATE]: Term 12 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a40db63d1c9748e99885c3264331297a, c7651a898f6142c793a606c219090c09; no voters: 836381137da64858b3e96d2b8f127242
12:12:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.620868 32417 raft_consensus.cc:2804] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 12 FOLLOWER]: Leader election won for term 12
12:12:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.622303 32417 raft_consensus.cc:697] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [term 12 LEADER]: Becoming Leader. State: Replica: a40db63d1c9748e99885c3264331297a, State: Running, Role: LEADER
12:12:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.623006 32417 consensus_queue.cc:237] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 12, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:12:58.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.630836 30026 catalog_manager.cc:5697] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a reported cstate change: term changed from 11 to 12, leader changed from 836381137da64858b3e96d2b8f127242 (127.29.71.129) to a40db63d1c9748e99885c3264331297a (127.29.71.130). New cstate: current_term: 12 leader_uuid: "a40db63d1c9748e99885c3264331297a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } health_report { overall_health: HEALTHY } } }
12:12:58.959 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 6a1f398f09354ae5af27df2891707f0c for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461' with partition [<start>, <end>)
12:12:58.960 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=6a1f398f09354ae5af27df2891707f0c@[836381137da64858b3e96d2b8f127242(127.29.71.129:44009),a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)[L],c7651a898f6142c793a606c219090c09(127.29.71.131:38005)]}]
12:12:58.960 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=6a1f398f09354ae5af27df2891707f0c, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1372), Traces: [0ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [2ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [2ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [13ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [14ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [14ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [31ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [31ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [33ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [33ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [72ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [73ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [74ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [74ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [188ms] refreshing cache from master, [188ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [191ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [193ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [194ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [194ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [428ms] refreshing cache from master, [428ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [432ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, [433ms] sending RPC to server 836381137da64858b3e96d2b8f127242, [434ms] received response from server 836381137da64858b3e96d2b8f127242: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [434ms] delaying RPC due to: Network error: Failed to connect to peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009): Connection refused: /127.29.71.129:44009, [1368ms] refreshing cache from master, [1368ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.71.190:46473, [1371ms] Sub RPC GetTableLocations: received response from server master-127.29.71.190:46473: OK, deferred=Deferred@1526783238(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$224/0x00007f6bac36b330@525b8b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@800548966 after retry RPC@1868350165) -> (continuation of Deferred@1641630068 after retry RPC@69366111) -> (continuation of Deferred@1061367752 after retry RPC@679217654) -> (continuation of Deferred@580720617 after retry RPC@1793493493) -> (continuation of Deferred@814010363 after retry RPC@1861439807), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2c4d8cc1 -> wakeup thread Time-limited test -> (continuation of Deferred@800548966 after retry RPC@1868350165) -> (continuation of Deferred@1641630068 after retry RPC@69366111) -> (continuation of Deferred@1061367752 after retry RPC@679217654) -> (continuation of Deferred@580720617 after retry RPC@1793493493) -> (continuation of Deferred@814010363 after retry RPC@1861439807))) after lookup
12:12:58.961 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] connecting to peer
12:12:58.962 [DEBUG - kudu-nio-0] (Connection.java:757) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] Successfully connected to peer
12:12:58.980 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x7b5176d0, L:/127.0.0.1:48672 - R:/127.29.71.130:36197] using TOKEN/null
12:12:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.991108 32360 raft_consensus.cc:1275] T 6a1f398f09354ae5af27df2891707f0c P c7651a898f6142c793a606c219090c09 [term 12 FOLLOWER]: Refusing update from remote peer a40db63d1c9748e99885c3264331297a: Log matching property violated. Preceding OpId in replica: term: 10 index: 36. Preceding OpId from leader: term: 12 index: 42. (index mismatch)
12:12:58.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.992699 32417 consensus_queue.cc:1048] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a [LEADER]: Connected to new peer: Peer: permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
12:12:58.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.993227 32121 consensus_peers.cc:597] T 6a1f398f09354ae5af27df2891707f0c P a40db63d1c9748e99885c3264331297a -> Peer 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009): Couldn't send request to peer 836381137da64858b3e96d2b8f127242. Status: Network error: Client connection negotiation failed: client connection to 127.29.71.129:44009: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:59.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.008499 32438 mvcc.cc:204] Tried to move back new op lower bound from 7305268327736987648 to 7305268327035891712. Current Snapshot: MvccSnapshot[applied={T|T < 7305268327736987648}]
12:12:59.055 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.71.129:44009
12:12:59.055 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.056506 29982 external_mini_cluster.cc:1372] Running /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu
12:12:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data
12:12:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/logs
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.129:44009
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.129
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42507
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.129
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.354830 32445 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.355367 32445 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.355849 32445 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.388214 32445 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:59.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.389047 32445 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.71.129
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.424788 32445 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.71.148:33751
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.71.129:44009
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.71.129
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42507
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.71.190:46473,127.29.71.189:40761,127.29.71.188:37701
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.71.129
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/logs
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:08 UTC on e7f111948823
12:12:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16857
12:12:59.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
12:12:59.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.426088 32445 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:59.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.427623 32445 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:59.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.439215 32452 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:00.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.440807 32453 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:00.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.724767 32454 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1279 milliseconds
12:13:00.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.727025 32455 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.729228 32445 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.289s	user 0.006s	sys 0.004s
12:13:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.729485 32445 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.289s	user 0.006s	sys 0.004s
12:13:00.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.729669 32445 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
12:13:00.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.730954 32445 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:13:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.733235 32445 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.734623 32445 hybrid_clock.cc:648] HybridClock initialized: now 1783512780734584 us; error 35 us; skew 500 ppm
12:13:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.735571 32445 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
12:13:00.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.744282 32445 webserver.cc:533] Webserver started at http://127.29.71.129:42507/ using document root <none> and password file <none>
12:13:00.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.745455 32445 fs_manager.cc:362] Metadata directory not provided
12:13:00.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.745723 32445 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:13:00.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.757122 32445 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.006s	sys 0.001s
12:13:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.763198 32468 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:13:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.764511 32445 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.006s	sys 0.000s
12:13:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.764968 32445 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data,/tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:13:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "836381137da64858b3e96d2b8f127242"
12:13:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:09 on dist-test-slave-tp5p"
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.767688 32445 fs_report.cc:389] FS layout report
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/wal
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/data
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:13:00.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.851676 32445 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:13:00.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.853287 32445 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:13:00.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.853861 32445 kserver.cc:163] Server-wide thread pool size limit: 3276
12:13:00.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.857015 32445 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:13:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.862969 32475 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
12:13:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.870937 32445 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:13:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.871178 32445 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
12:13:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.871402 32445 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:13:00.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.876056 32445 ts_tablet_manager.cc:616] Registered 1 tablets
12:13:00.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.876237 32445 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
12:13:00.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.876684 32475 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap starting.
12:13:00.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.945974 32475 log.cc:826] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Log is configured to *not* fsync() on all Append() calls
12:13:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.060163 32445 rpc_server.cc:307] RPC server started. Bound to: 127.29.71.129:44009
12:13:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.060261 32583 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.71.129:44009 every 8 connection(s)
12:13:01.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.062119 32475 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:13:01.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.063058 32475 tablet_bootstrap.cc:492] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Bootstrap complete.
12:13:01.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.063704 32445 server_base.cc:1193] Dumped server information to /tmp/dist-test-task8wlbrb/test-tmp/mini-kudu-cluster4916948434544382030/ts-0/data/info.pb
12:13:01.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.064752 32475 ts_tablet_manager.cc:1403] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Time spent bootstrapping tablet: real 0.188s	user 0.143s	sys 0.037s
12:13:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.072644 29982 external_mini_cluster.cc:1434] Started /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu as pid 32445
12:13:01.073 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:13:01.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.090710 32475 raft_consensus.cc:359] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 11 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:13:01.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.094069 32475 raft_consensus.cc:740] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 11 FOLLOWER]: Becoming Follower/Learner. State: Replica: 836381137da64858b3e96d2b8f127242, State: Initialized, Role: FOLLOWER
12:13:01.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.095372 32475 consensus_queue.cc:260] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7651a898f6142c793a606c219090c09" member_type: VOTER last_known_addr { host: "127.29.71.131" port: 38005 } } peers { permanent_uuid: "836381137da64858b3e96d2b8f127242" member_type: VOTER last_known_addr { host: "127.29.71.129" port: 44009 } } peers { permanent_uuid: "a40db63d1c9748e99885c3264331297a" member_type: VOTER last_known_addr { host: "127.29.71.130" port: 36197 } }
12:13:01.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.104992 32475 ts_tablet_manager.cc:1434] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242: Time spent starting tablet: real 0.040s	user 0.030s	sys 0.008s
12:13:01.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.130880 32584 heartbeater.cc:344] Connected to a master server at 127.29.71.188:37701
12:13:01.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.131317 32584 heartbeater.cc:461] Registering TS with master...
12:13:01.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.132311 32585 heartbeater.cc:344] Connected to a master server at 127.29.71.189:40761
12:13:01.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.132606 32584 heartbeater.cc:507] Master 127.29.71.188:37701 requested a full tablet report, sending...
12:13:01.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.132714 32585 heartbeater.cc:461] Registering TS with master...
12:13:01.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.133719 32585 heartbeater.cc:507] Master 127.29.71.189:40761 requested a full tablet report, sending...
12:13:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.137992 30102 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:13:01.140 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:13:01.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.145498 30182 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:13:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.144037 32586 heartbeater.cc:344] Connected to a master server at 127.29.71.190:46473
12:13:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.148290 32586 heartbeater.cc:461] Registering TS with master...
12:13:01.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.149214 32586 heartbeater.cc:507] Master 127.29.71.190:46473 requested a full tablet report, sending...
12:13:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.152704 30026 ts_manager.cc:194] Re-registered known tserver with Master: 836381137da64858b3e96d2b8f127242 (127.29.71.129:44009)
12:13:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.155407 30026 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.71.129:54961
12:13:01.156 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512734461, tablet=6a1f398f09354ae5af27df2891707f0c, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:13:01.163 [DEBUG - main] (Connection.java:575) [peer c7651a898f6142c793a606c219090c09(127.29.71.131:38005)] disconnecting while in state TERMINATED
12:13:01.164 [DEBUG - main] (Connection.java:575) [peer 836381137da64858b3e96d2b8f127242(127.29.71.129:44009)] disconnecting while in state TERMINATED
12:13:01.164 [DEBUG - main] (Connection.java:575) [peer master-127.29.71.190:46473(127.29.71.190:46473)] disconnecting while in state READY
12:13:01.164 [DEBUG - main] (Connection.java:575) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] disconnecting while in state READY
12:13:01.165 [DEBUG - main] (Connection.java:575) [peer master-127.29.71.189:40761(127.29.71.189:40761)] disconnecting while in state READY
12:13:01.165 [DEBUG - main] (Connection.java:575) [peer master-127.29.71.188:37701(127.29.71.188:37701)] disconnecting while in state READY
12:13:01.167 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.29.71.188:37701(127.29.71.188:37701)] handling channelInactive
12:13:01.168 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.29.71.188:37701(127.29.71.188:37701)] cleaning up while in state READY due to: connection closed
12:13:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.167922 32538 raft_consensus.cc:3060] T 6a1f398f09354ae5af27df2891707f0c P 836381137da64858b3e96d2b8f127242 [term 11 FOLLOWER]: Advancing to term 12
12:13:01.169 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.29.71.189:40761(127.29.71.189:40761)] handling channelInactive
12:13:01.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.71.189:40761(127.29.71.189:40761)] cleaning up while in state READY due to: connection closed
12:13:01.171 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
12:13:01.172 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.71.190:46473(127.29.71.190:46473)] handling channelInactive
12:13:01.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.71.190:46473(127.29.71.190:46473)] cleaning up while in state READY due to: connection closed
12:13:01.172 [DEBUG - kudu-nio-0] (Connection.java:239) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] handling channelInactive
12:13:01.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a40db63d1c9748e99885c3264331297a(127.29.71.130:36197)] cleaning up while in state READY due to: connection closed
12:13:01.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.178470 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 32445
12:13:01.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.213624 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 32098
12:13:01.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.239647 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 32274
12:13:01.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.264700 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 29995
12:13:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.291361 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 30065
12:13:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:01.316046 29982 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task8wlbrb/build/tsan/bin/kudu with pid 30140
12:13:02.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:13:02Z chronyd exiting

Time: 63.492

OK (1 test)

