JUnit version 4.13.2
.21:04:19.014 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783544659006
21:04:19.076 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
21:04:19.114 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:04:19.128 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrAZmyJ/test-tmp
21:04:19.161 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:04:19.165 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrAZmyJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:04:19.375 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

21:04:19.524 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:04:19.525 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

21:04:19.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.529217  9644 mini_oidc.cc:150] Starting JWKS server
21:04:19.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.532701  9644 webserver.cc:534] Webserver started at http://127.0.0.1:38851/ using document root <none> and password file <none>
21:04:19.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.533202  9644 mini_oidc.cc:186] Starting OIDC Discovery server
21:04:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.535395  9644 webserver.cc:534] Webserver started at http://127.0.0.1:34225/ using document root <none> and password file <none>
21:04:19.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:04:19Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:04:19.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:04:19Z Disabled control of system clock
21:04:19.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.589911  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:19.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:19.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/wal
21:04:19.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/data
21:04:19.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:19.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/logs
21:04:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/data/info.pb
21:04:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:19.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:19.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:19.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:19.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:19.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.62:44527
21:04:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.62
21:04:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38851/jwks.json with env {}
21:04:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.899926  9656 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.900722  9656 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.901551  9656 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.938428  9656 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:04:19.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.938719  9656 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:19.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.938941  9656 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:19.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.939128  9656 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:04:19.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.939313  9656 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:19.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.939471  9656 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38851/jwks.json
21:04:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.977174  9656 master_runner.cc:387] Master server non-default flags:
21:04:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/data
21:04:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/wal
21:04:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.62:44527
21:04:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38851/jwks.json
21:04:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/data/info.pb
21:04:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.62
21:04:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/logs
21:04:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:19.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:19.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:19.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:19.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:19.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.978662  9656 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.980372  9656 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:19.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.992667  9662 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.992673  9663 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.131366  9664 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1137 milliseconds
21:04:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.136082  9656 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.144s	user 0.426s	sys 0.715s
21:04:21.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.136412  9656 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.145s	user 0.426s	sys 0.715s
21:04:21.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.136855  9665 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:21.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.136900  9656 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:04:21.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.138222  9656 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.140899  9656 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.142282  9656 hybrid_clock.cc:648] HybridClock initialized: now 1783544661142197 us; error 89 us; skew 500 ppm
21:04:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.143220  9656 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:21.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.150709  9656 webserver.cc:534] Webserver started at http://127.9.107.62:41799/ using document root <none> and password file <none>
21:04:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.151908  9656 fs_manager.cc:362] Metadata directory not provided
21:04:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.152153  9656 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.152716  9656 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:21.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.157148  9656 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/data/instance:
21:04:21.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1b2eaf6402b94c738c2ee9252c738ec5"
21:04:21.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:21 on dist-test-slave-qc6t"
21:04:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.158601  9656 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/wal/instance:
21:04:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1b2eaf6402b94c738c2ee9252c738ec5"
21:04:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:21 on dist-test-slave-qc6t"
21:04:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.168361  9656 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.005s	sys 0.005s
21:04:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.174618  9672 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:21.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.176194  9656 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.002s
21:04:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.176574  9656 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/wal
21:04:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1b2eaf6402b94c738c2ee9252c738ec5"
21:04:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:21 on dist-test-slave-qc6t"
21:04:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.176999  9656 fs_report.cc:389] FS layout report
21:04:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/wal
21:04:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/wal
21:04:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/data/data
21:04:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:21.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:21.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:21.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:21.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.244167  9656 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.245687  9656 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.246099  9656 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.316882  9656 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.62:44527
21:04:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.316977  9723 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.62:44527 every 8 connection(s)
21:04:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.319525  9656 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/data/info.pb
21:04:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.325789  9724 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.326220  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 9656
21:04:21.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.326699  9644 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-0/wal/instance
21:04:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.331617  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/wal
21:04:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/data
21:04:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/logs
21:04:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/data/info.pb
21:04:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.61:39893
21:04:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.61
21:04:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38851/jwks.json with env {}
21:04:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.345574  9724 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } has no permanent_uuid. Determining permanent_uuid...
21:04:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.374694  9724 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } has no permanent_uuid. Determining permanent_uuid...
21:04:21.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.378203  9675 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.61:39893: connect: Connection refused (error 111)
21:04:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.384527  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.61:39893: Network error: Client connection negotiation failed: client connection to 127.9.107.61:39893: connect: Connection refused (error 111)
21:04:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.434197  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } attempt: 1
21:04:21.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.439420  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.61:39893: Network error: Client connection negotiation failed: client connection to 127.9.107.61:39893: connect: Connection refused (error 111)
21:04:21.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.507963  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } attempt: 2
21:04:21.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.512343  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.61:39893: Network error: Client connection negotiation failed: client connection to 127.9.107.61:39893: connect: Connection refused (error 111)
21:04:21.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.603802  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } attempt: 3
21:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.608814  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.61:39893: Network error: Client connection negotiation failed: client connection to 127.9.107.61:39893: connect: Connection refused (error 111)
21:04:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.668857  9726 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.669504  9726 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.669988  9726 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:21.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.703452  9726 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:04:21.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.703841  9726 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:21.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.704054  9726 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:21.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.704242  9726 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:04:21.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.704511  9726 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:21.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.704743  9726 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38851/jwks.json
21:04:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.742640  9726 master_runner.cc:387] Master server non-default flags:
21:04:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/data
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/wal
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.61:39893
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38851/jwks.json
21:04:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/data/info.pb
21:04:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.61
21:04:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/logs
21:04:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:21.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:21.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:21.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:21.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:21.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:21.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:21.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:21.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:21.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.743995  9726 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:21.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.746004  9726 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:21.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.752226  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } attempt: 4
21:04:21.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.759838  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.61:39893: Network error: Client connection negotiation failed: client connection to 127.9.107.61:39893: connect: Connection refused (error 111)
21:04:21.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.761402  9735 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.059331  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } attempt: 5
21:04:22.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:22.064450  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.61:39893: Network error: Client connection negotiation failed: client connection to 127.9.107.61:39893: connect: Connection refused (error 111)
21:04:22.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.611863  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } attempt: 6
21:04:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:22.622251  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.61:39893: Network error: Client connection negotiation failed: client connection to 127.9.107.61:39893: connect: Connection refused (error 111)
21:04:22.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.766111  9736 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:22.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:21.767064  9738 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:22.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.767683  9726 server_base.cc:1061] running on GCE node
21:04:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.953003  9726 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.956175  9726 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:22.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.957662  9726 hybrid_clock.cc:648] HybridClock initialized: now 1783544662957570 us; error 84 us; skew 500 ppm
21:04:22.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.958770  9726 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:22.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.965049  9726 webserver.cc:534] Webserver started at http://127.9.107.61:39403/ using document root <none> and password file <none>
21:04:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.966189  9726 fs_manager.cc:362] Metadata directory not provided
21:04:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.966418  9726 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.967000  9726 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.971609  9726 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/data/instance:
21:04:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "019339474fe74e2698e7609aa80f5318"
21:04:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:22 on dist-test-slave-qc6t"
21:04:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.973059  9726 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/wal/instance:
21:04:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "019339474fe74e2698e7609aa80f5318"
21:04:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:22 on dist-test-slave-qc6t"
21:04:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.980571  9726 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.000s
21:04:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.987584  9747 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.988771  9726 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.001s
21:04:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.989090  9726 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/wal
21:04:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "019339474fe74e2698e7609aa80f5318"
21:04:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:22 on dist-test-slave-qc6t"
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.989463  9726 fs_report.cc:389] FS layout report
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/wal
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/wal
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/data/data
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:22.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:22.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:22.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:22.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:22.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:22.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.046607  9726 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:23.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.048080  9726 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:23.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.048496  9726 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.120199  9726 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.61:39893
21:04:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.120296  9798 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.61:39893 every 8 connection(s)
21:04:23.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.123034  9726 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/data/info.pb
21:04:23.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.126348  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 9726
21:04:23.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.126789  9644 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-1/wal/instance
21:04:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.130370  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/wal
21:04:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/data
21:04:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/logs
21:04:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/data/info.pb
21:04:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.60:46817
21:04:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.60
21:04:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38851/jwks.json with env {}
21:04:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.131006  9799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.149876  9799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } has no permanent_uuid. Determining permanent_uuid...
21:04:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.176867  9799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } has no permanent_uuid. Determining permanent_uuid...
21:04:23.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.192088  9799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } has no permanent_uuid. Determining permanent_uuid...
21:04:23.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.195216  9750 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.199632  9799 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:23.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.249172  9799 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 1
21:04:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.253633  9799 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.322094  9799 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 2
21:04:23.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.327575  9799 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.419061  9799 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 3
21:04:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.423421  9799 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.466604  9801 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.467165  9801 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.467600  9801 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:23.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.501240  9801 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:04:23.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.501631  9801 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:23.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.502141  9801 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:23.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.502439  9801 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:04:23.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.502760  9801 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:23.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.502939  9801 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38851/jwks.json
21:04:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.539525  9801 master_runner.cc:387] Master server non-default flags:
21:04:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/data
21:04:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/wal
21:04:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.60:46817
21:04:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38851/jwks.json
21:04:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/data/info.pb
21:04:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.60
21:04:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/logs
21:04:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.540966  9801 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.543181  9801 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.556382  9812 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.566852  9799 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 4
21:04:23.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.572618  9799 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:23.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.682690  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } attempt: 7
21:04:23.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.714778  9724 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } has no permanent_uuid. Determining permanent_uuid...
21:04:23.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.740948  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:23.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.799494  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 1
21:04:23.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.806804  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:23.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.872324  9799 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 5
21:04:23.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.888515  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 2
21:04:23.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.892932  9799 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:23.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.900229  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:23.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:23.986255  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 3
21:04:23.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.996028  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:24.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.136519  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 4
21:04:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.145134  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.428931  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 5
21:04:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.438999  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:24.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.440478  9799 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 6
21:04:24.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.466154  9799 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:24.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:23.557548  9811 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.896078  9814 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:24.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.899685  9801 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.344s	user 0.421s	sys 0.893s
21:04:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.899971  9801 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.345s	user 0.421s	sys 0.893s
21:04:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.900059  9813 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1342 milliseconds
21:04:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.900233  9801 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:04:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.901476  9801 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.904292  9801 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:24.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.905674  9801 hybrid_clock.cc:648] HybridClock initialized: now 1783544664905607 us; error 81 us; skew 500 ppm
21:04:24.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.906591  9801 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:24.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.914286  9801 webserver.cc:534] Webserver started at http://127.9.107.60:42735/ using document root <none> and password file <none>
21:04:24.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.915484  9801 fs_manager.cc:362] Metadata directory not provided
21:04:24.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.915709  9801 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.916262  9801 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:24.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.921079  9801 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/data/instance:
21:04:24.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b3da7bbb3d3f42d38b023800c742dff0"
21:04:24.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:24 on dist-test-slave-qc6t"
21:04:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.922514  9801 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/wal/instance:
21:04:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b3da7bbb3d3f42d38b023800c742dff0"
21:04:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:24 on dist-test-slave-qc6t"
21:04:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.932222  9801 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.005s	sys 0.005s
21:04:24.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.938586  9825 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:24.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.941360  9801 fs_manager.cc:730] Time spent opening block manager: real 0.006s	user 0.003s	sys 0.002s
21:04:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.941788  9801 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/wal
21:04:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b3da7bbb3d3f42d38b023800c742dff0"
21:04:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:24 on dist-test-slave-qc6t"
21:04:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.942234  9801 fs_report.cc:389] FS layout report
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/wal
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/wal
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/data/data
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:24.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:24.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:24.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:24.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.991524  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 6
21:04:24.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.998319  9724 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.107.60:46817: Network error: Client connection negotiation failed: client connection to 127.9.107.60:46817: connect: Connection refused (error 111)
21:04:25.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.024992  9801 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:25.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.026965  9801 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:25.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.027462  9801 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:25.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.099038  9801 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.60:46817
21:04:25.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.099140  9878 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.60:46817 every 8 connection(s)
21:04:25.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.101925  9801 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/data/info.pb
21:04:25.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.107800  9879 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.111322  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 9801
21:04:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.111663  9644 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/master-2/wal/instance
21:04:25.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.121222  9879 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } has no permanent_uuid. Determining permanent_uuid...
21:04:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.160372  9879 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } has no permanent_uuid. Determining permanent_uuid...
21:04:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.171515  9879 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } has no permanent_uuid. Determining permanent_uuid...
21:04:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.188587  9879 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0: Bootstrap starting.
21:04:25.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.195237  9879 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0: Neither blocks nor log segments found. Creating new log.
21:04:25.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.197160  9879 log.cc:826] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0: Log is configured to *not* fsync() on all Append() calls
21:04:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.202518  9879 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0: No bootstrap required, opened a new log
21:04:25.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.222481  9879 raft_consensus.cc:359] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } }
21:04:25.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.223169  9879 raft_consensus.cc:385] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:25.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.223439  9879 raft_consensus.cc:740] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b3da7bbb3d3f42d38b023800c742dff0, State: Initialized, Role: FOLLOWER
21:04:25.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.224679  9879 consensus_queue.cc:260] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } }
21:04:25.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.228106  9888 sys_catalog.cc:455] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } } }
21:04:25.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.228950  9888 sys_catalog.cc:458] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [sys.catalog]: This master's current role is: FOLLOWER
21:04:25.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.229822  9879 sys_catalog.cc:565] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [sys.catalog]: configured and running, proceeding with master startup.
21:04:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:25.259078  9899 catalog_manager.cc:1593] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:04:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:25.259938  9899 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:04:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:25.283330  9833 tablet.cc:2406] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.254025  9801 master_runner.cc:187] Error getting master registration for 127.9.107.62:44527: OK, instance_id { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" instance_seqno: 1783544661298005 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:04:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.288153  9801 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.431195  9888 raft_consensus.cc:493] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.431803  9888 raft_consensus.cc:515] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } }
21:04:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.435249  9888 leader_election.cc:290] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1b2eaf6402b94c738c2ee9252c738ec5 (127.9.107.62:44527), 019339474fe74e2698e7609aa80f5318 (127.9.107.61:39893)
21:04:25.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.526592  9799 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 7
21:04:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.549104  9799 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318: Bootstrap starting.
21:04:25.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.556866  9799 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318: Neither blocks nor log segments found. Creating new log.
21:04:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.559405  9799 log.cc:826] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318: Log is configured to *not* fsync() on all Append() calls
21:04:25.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.564875  9799 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318: No bootstrap required, opened a new log
21:04:25.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.594738  9799 raft_consensus.cc:359] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } }
21:04:25.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.596105  9799 raft_consensus.cc:385] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:25.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.596698  9799 raft_consensus.cc:740] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 019339474fe74e2698e7609aa80f5318, State: Initialized, Role: FOLLOWER
21:04:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.598030  9799 consensus_queue.cc:260] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } }
21:04:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.600939  9903 sys_catalog.cc:455] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } } }
21:04:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.601776  9903 sys_catalog.cc:458] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [sys.catalog]: This master's current role is: FOLLOWER
21:04:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.602957  9799 sys_catalog.cc:565] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [sys.catalog]: configured and running, proceeding with master startup.
21:04:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.603097  9774 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b3da7bbb3d3f42d38b023800c742dff0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "019339474fe74e2698e7609aa80f5318" is_pre_election: true
21:04:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.603966  9774 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b3da7bbb3d3f42d38b023800c742dff0 in term 0.
21:04:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.605446  9830 leader_election.cc:304] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 019339474fe74e2698e7609aa80f5318, b3da7bbb3d3f42d38b023800c742dff0; no voters: 
21:04:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.606159  9888 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.606526  9888 raft_consensus.cc:493] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:25.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.606860  9888 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 0 FOLLOWER]: Advancing to term 1
21:04:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.613499  9888 raft_consensus.cc:515] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } }
21:04:25.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.615521  9888 leader_election.cc:290] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [CANDIDATE]: Term 1 election: Requested vote from peers 1b2eaf6402b94c738c2ee9252c738ec5 (127.9.107.62:44527), 019339474fe74e2698e7609aa80f5318 (127.9.107.61:39893)
21:04:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.617664  9774 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b3da7bbb3d3f42d38b023800c742dff0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "019339474fe74e2698e7609aa80f5318"
21:04:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.618211  9774 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [term 0 FOLLOWER]: Advancing to term 1
21:04:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.625635  9774 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b3da7bbb3d3f42d38b023800c742dff0 in term 1.
21:04:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.634560  9830 leader_election.cc:304] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 019339474fe74e2698e7609aa80f5318, b3da7bbb3d3f42d38b023800c742dff0; no voters: 
21:04:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.635366  9888 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 1 FOLLOWER]: Leader election won for term 1
21:04:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.642256  9888 raft_consensus.cc:697] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [term 1 LEADER]: Becoming Leader. State: Replica: b3da7bbb3d3f42d38b023800c742dff0, State: Running, Role: LEADER
21:04:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.643601  9888 consensus_queue.cc:237] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } }
21:04:25.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.652588  9909 sys_catalog.cc:455] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b3da7bbb3d3f42d38b023800c742dff0. Latest consensus state: current_term: 1 leader_uuid: "b3da7bbb3d3f42d38b023800c742dff0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } } }
21:04:25.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.653172  9909 sys_catalog.cc:458] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [sys.catalog]: This master's current role is: LEADER
21:04:25.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.660123  9914 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
21:04:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.672627  9914 catalog_manager.cc:1519] Initializing Kudu cluster ID...
21:04:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.707870  9774 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [term 1 FOLLOWER]: Refusing update from remote peer b3da7bbb3d3f42d38b023800c742dff0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.679396  9726 master_runner.cc:187] Error getting master registration for 127.9.107.62:44527: OK, instance_id { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" instance_seqno: 1783544661298005 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:04:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.710381  9909 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.711589  9726 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:04:25.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:25.719264  9917 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:04:25.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:25.720191  9917 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:04:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.744262  9903 sys_catalog.cc:455] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b3da7bbb3d3f42d38b023800c742dff0. Latest consensus state: current_term: 1 leader_uuid: "b3da7bbb3d3f42d38b023800c742dff0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } } }
21:04:25.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.746810  9903 sys_catalog.cc:458] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [sys.catalog]: This master's current role is: FOLLOWER
21:04:25.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.754061  9923 sys_catalog.cc:455] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b3da7bbb3d3f42d38b023800c742dff0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } } }
21:04:25.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.755295  9923 sys_catalog.cc:458] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318 [sys.catalog]: This master's current role is: FOLLOWER
21:04:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.770124  9920 mvcc.cc:204] Tried to move back new op lower bound from 7305398950673719296 to 7305398950488006656. Current Snapshot: MvccSnapshot[applied={T|T < 7305398950673719296}]
21:04:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.770179  9909 sys_catalog.cc:455] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b3da7bbb3d3f42d38b023800c742dff0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } } }
21:04:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.770885  9909 sys_catalog.cc:458] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [sys.catalog]: This master's current role is: LEADER
21:04:25.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.781558  9914 catalog_manager.cc:1382] Generated new cluster ID: 5949040c8aa040e6a24cf0cc0a1b2bdc
21:04:25.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.781912  9914 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
21:04:25.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.841033  9914 catalog_manager.cc:1405] Generated new certificate authority record
21:04:25.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.843801  9914 catalog_manager.cc:1539] Loading token signing keys...
21:04:25.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.879621  9914 catalog_manager.cc:6069] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0: Generated new TSK 0
21:04:25.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.884342  9914 catalog_manager.cc:1549] Initializing in-progress tserver states...
21:04:25.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.937765  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:25.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:25.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:25.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data
21:04:25.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:25.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/logs
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.1:0
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.1
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.1
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:25.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:26.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.031759  9724 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } attempt: 7
21:04:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.077302  9724 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5: Bootstrap starting.
21:04:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.084791  9724 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5: Neither blocks nor log segments found. Creating new log.
21:04:26.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.087916  9724 log.cc:826] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5: Log is configured to *not* fsync() on all Append() calls
21:04:26.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.099246  9724 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5: No bootstrap required, opened a new log
21:04:26.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.132144  9724 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } }
21:04:26.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.133520  9724 raft_consensus.cc:385] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:26.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.134199  9724 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1b2eaf6402b94c738c2ee9252c738ec5, State: Initialized, Role: FOLLOWER
21:04:26.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.135690  9724 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } }
21:04:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.151088  9928 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } } }
21:04:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.151911  9928 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [sys.catalog]: This master's current role is: FOLLOWER
21:04:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.152509  9724 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [sys.catalog]: configured and running, proceeding with master startup.
21:04:26.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.152745  9698 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b3da7bbb3d3f42d38b023800c742dff0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1b2eaf6402b94c738c2ee9252c738ec5"
21:04:26.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.153505  9698 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [term 0 FOLLOWER]: Advancing to term 1
21:04:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.158649  9699 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b3da7bbb3d3f42d38b023800c742dff0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" is_pre_election: true
21:04:26.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.169667  9698 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b3da7bbb3d3f42d38b023800c742dff0 in term 1.
21:04:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.171885  9697 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [term 1 FOLLOWER]: Refusing update from remote peer b3da7bbb3d3f42d38b023800c742dff0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:26.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.181450  9888 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.260493  9938 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b3da7bbb3d3f42d38b023800c742dff0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } } }
21:04:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.265398  9938 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [sys.catalog]: This master's current role is: FOLLOWER
21:04:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.264283  9928 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b3da7bbb3d3f42d38b023800c742dff0. Latest consensus state: current_term: 1 leader_uuid: "b3da7bbb3d3f42d38b023800c742dff0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } } }
21:04:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.266440  9928 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5 [sys.catalog]: This master's current role is: FOLLOWER
21:04:26.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.292528  9945 catalog_manager.cc:1294] Loaded cluster ID: 5949040c8aa040e6a24cf0cc0a1b2bdc
21:04:26.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.293416  9945 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5: loading cluster ID for follower catalog manager: success
21:04:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.301532  9945 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5: acquiring CA information for follower catalog manager: success
21:04:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.308261  9945 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 1b2eaf6402b94c738c2ee9252c738ec5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:26.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.500376  9925 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:26.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.501708  9925 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:26.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.503216  9925 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.564397  9925 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.566227  9925 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.1
21:04:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.632728  9925 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data
21:04:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.1:0
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.1
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.1
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/logs
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:26.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:26.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.637576  9925 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:26.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.640087  9925 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:26.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.671789  9952 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:26.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.726832  9917 catalog_manager.cc:1294] Loaded cluster ID: 5949040c8aa040e6a24cf0cc0a1b2bdc
21:04:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.727128  9917 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318: loading cluster ID for follower catalog manager: success
21:04:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.741691  9917 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318: acquiring CA information for follower catalog manager: success
21:04:26.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.747849  9917 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 019339474fe74e2698e7609aa80f5318: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:26.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.779572  9888 sys_catalog.cc:455] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b3da7bbb3d3f42d38b023800c742dff0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2eaf6402b94c738c2ee9252c738ec5" member_type: VOTER last_known_addr { host: "127.9.107.62" port: 44527 } } peers { permanent_uuid: "019339474fe74e2698e7609aa80f5318" member_type: VOTER last_known_addr { host: "127.9.107.61" port: 39893 } } peers { permanent_uuid: "b3da7bbb3d3f42d38b023800c742dff0" member_type: VOTER last_known_addr { host: "127.9.107.60" port: 46817 } } }
21:04:26.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.780280  9888 sys_catalog.cc:458] T 00000000000000000000000000000000 P b3da7bbb3d3f42d38b023800c742dff0 [sys.catalog]: This master's current role is: LEADER
21:04:28.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.688236  9956 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:28.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.688802  9925 server_base.cc:1061] running on GCE node
21:04:28.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.686697  9953 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:28.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.270201  9925 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:28.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.283463  9925 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.285686  9925 hybrid_clock.cc:648] HybridClock initialized: now 1783544668285502 us; error 54 us; skew 500 ppm
21:04:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.287848  9925 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:28.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.313392  9925 webserver.cc:534] Webserver started at http://127.9.107.1:44535/ using document root <none> and password file <none>
21:04:28.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.315155  9925 fs_manager.cc:362] Metadata directory not provided
21:04:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.315789  9925 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.316546  9925 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.323525  9925 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/instance:
21:04:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa68dbc27a334b3bb725578a111601e4"
21:04:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:28 on dist-test-slave-qc6t"
21:04:28.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.325917  9925 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal/instance:
21:04:28.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa68dbc27a334b3bb725578a111601e4"
21:04:28.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:28 on dist-test-slave-qc6t"
21:04:28.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.343626  9925 fs_manager.cc:696] Time spent creating directory manager: real 0.017s	user 0.013s	sys 0.000s
21:04:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.357460  9964 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:28.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.359369  9925 fs_manager.cc:730] Time spent opening block manager: real 0.011s	user 0.010s	sys 0.001s
21:04:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.360102  9925 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa68dbc27a334b3bb725578a111601e4"
21:04:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:28 on dist-test-slave-qc6t"
21:04:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.360903  9925 fs_report.cc:389] FS layout report
21:04:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/data
21:04:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.448809  9925 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.450819  9925 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.451304  9925 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.455487  9925 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.460913  9925 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.461140  9925 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:04:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.461431  9925 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.461671  9925 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:04:28.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.671790  9925 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.1:34711
21:04:28.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.671904 10076 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.1:34711 every 8 connection(s)
21:04:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.674940  9925 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:04:28.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.682582  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 9925
21:04:28.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.683094  9644 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal/instance
21:04:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.733886  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/logs
21:04:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.2:0
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.2
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.2
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:28.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.809437 10077 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:04:28.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.809942 10077 heartbeater.cc:461] Registering TS with master...
21:04:28.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.811667 10077 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:04:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.816430  9844 ts_manager.cc:194] Registered new tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:04:28.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.820581  9844 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.1:48249
21:04:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.836448 10082 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:04:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.837626 10082 heartbeater.cc:461] Registering TS with master...
21:04:28.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.838747 10082 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:04:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.844527  9689 ts_manager.cc:194] Registered new tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:04:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.846253 10078 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:04:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.846679 10078 heartbeater.cc:461] Registering TS with master...
21:04:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.847586 10078 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:04:28.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.850710  9764 ts_manager.cc:194] Registered new tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:04:29.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.211045 10081 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:29.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.211491 10081 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:29.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.211899 10081 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:29.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.244562 10081 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:29.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.245404 10081 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.2
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.280581 10081 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.2:0
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.2
21:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.2
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/logs
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.281915 10081 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:29.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.283861 10081 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:29.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.299047 10093 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:29.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.828907 10077 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:04:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.700500 10092 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 10081
21:04:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.299232 10094 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.882412 10081 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.583s	user 0.606s	sys 0.964s
21:04:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.882754 10081 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.584s	user 0.607s	sys 0.964s
21:04:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.883651 10096 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.885749 10095 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1582 milliseconds
21:04:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.885803 10081 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:04:30.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.887014 10081 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:30.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.889149 10081 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.890552 10081 hybrid_clock.cc:648] HybridClock initialized: now 1783544670890502 us; error 49 us; skew 500 ppm
21:04:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.891417 10081 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:30.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.897527 10081 webserver.cc:534] Webserver started at http://127.9.107.2:39083/ using document root <none> and password file <none>
21:04:30.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.898458 10081 fs_manager.cc:362] Metadata directory not provided
21:04:30.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.898674 10081 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:30.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.899183 10081 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.903712 10081 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/instance:
21:04:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18c80cf9fb014eb9bc13a9e176e05a00"
21:04:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:30 on dist-test-slave-qc6t"
21:04:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.905112 10081 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal/instance:
21:04:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18c80cf9fb014eb9bc13a9e176e05a00"
21:04:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:30 on dist-test-slave-qc6t"
21:04:30.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.912175 10081 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.000s
21:04:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.917968 10105 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.919474 10081 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
21:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.919823 10081 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18c80cf9fb014eb9bc13a9e176e05a00"
21:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:30 on dist-test-slave-qc6t"
21:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.920215 10081 fs_report.cc:389] FS layout report
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/data
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:30.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.978096 10081 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.979753 10081 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.980243 10081 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.982936 10081 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.987344 10081 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.987564 10081 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:04:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.987851 10081 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.988072 10081 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:04:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.149261 10081 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.2:42877
21:04:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.149403 10217 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.2:42877 every 8 connection(s)
21:04:31.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.152793 10081 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:04:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.160384  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 10081
21:04:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.161314  9644 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal/instance
21:04:31.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.186939  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/logs
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.3:0
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.3
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.3
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.197099 10220 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:04:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.197503 10218 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:04:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.197595 10219 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:04:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.197674 10220 heartbeater.cc:461] Registering TS with master...
21:04:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.197989 10219 heartbeater.cc:461] Registering TS with master...
21:04:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.198026 10218 heartbeater.cc:461] Registering TS with master...
21:04:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.199297 10220 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:04:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.199458 10218 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:04:31.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.200232 10219 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:04:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.202811  9844 ts_manager.cc:194] Registered new tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.202903  9689 ts_manager.cc:194] Registered new tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.203204  9764 ts_manager.cc:194] Registered new tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.204444  9844 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.2:46681
21:04:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:31.519451 10228 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:31.519930 10228 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:31.520329 10228 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:31.553580 10228 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:31.554548 10228 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.3
21:04:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.590791 10228 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data
21:04:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.3:0
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.3
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.3
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/logs
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.592628 10228 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:31.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.594619 10228 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:31.607714 10235 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.208173 10218 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:04:33.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:33.140188 10228 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.533s	user 0.610s	sys 0.903s
21:04:33.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:33.140600 10228 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.533s	user 0.611s	sys 0.903s
21:04:33.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:31.608073 10236 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:33.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:33.010897 10234 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 10228
21:04:33.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:33.142993 10238 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:33.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:33.150652 10237 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1531 milliseconds
21:04:33.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.150727 10228 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:04:33.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.153191 10228 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:33.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.155900 10228 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:33.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.159606 10228 hybrid_clock.cc:648] HybridClock initialized: now 1783544673159232 us; error 376 us; skew 500 ppm
21:04:33.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.160657 10228 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:33.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.168325 10228 webserver.cc:534] Webserver started at http://127.9.107.3:38725/ using document root <none> and password file <none>
21:04:33.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.169394 10228 fs_manager.cc:362] Metadata directory not provided
21:04:33.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.169622 10228 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.170136 10228 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:33.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.174429 10228 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/instance:
21:04:33.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:04:33.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:33 on dist-test-slave-qc6t"
21:04:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.175745 10228 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal/instance:
21:04:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:04:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:33 on dist-test-slave-qc6t"
21:04:33.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.183696 10228 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.000s
21:04:33.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.189383 10246 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:33.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.190460 10228 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
21:04:33.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.190807 10228 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:33.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:04:33.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:33 on dist-test-slave-qc6t"
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.191139 10228 fs_report.cc:389] FS layout report
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/data
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:33.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.248529 10228 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:33.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.250161 10228 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:33.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.250664 10228 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:33.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.253857 10228 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:33.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.258241 10228 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.258451 10228 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:04:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.258636 10228 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.258761 10228 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:04:33.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.420379 10228 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.3:42267
21:04:33.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.420567 10358 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.3:42267 every 8 connection(s)
21:04:33.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.423136 10228 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:04:33.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.425575  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 10228
21:04:33.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.426087  9644 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal/instance
21:04:33.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.476428 10361 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:04:33.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.476990 10361 heartbeater.cc:461] Registering TS with master...
21:04:33.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.478413 10361 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:04:33.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.479457 10360 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:04:33.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.479779 10360 heartbeater.cc:461] Registering TS with master...
21:04:33.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.480690 10360 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:04:33.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.484400  9764 ts_manager.cc:194] Registered new tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.486292  9689 ts_manager.cc:194] Registered new tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:33.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.490073 10359 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:04:33.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.490437 10359 heartbeater.cc:461] Registering TS with master...
21:04:33.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.491223 10359 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:04:33.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.493546  9844 ts_manager.cc:194] Registered new tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.495167  9844 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.3:58219
21:04:33.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.515852  9644 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
21:04:33.527 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:04:33.528 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

21:04:33.537 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.9.107.62"
      port: 44527
    }
    bound_http_address {
      host: "127.9.107.62"
      port: 41799
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.9.107.61"
      port: 39893
    }
    bound_http_address {
      host: "127.9.107.61"
      port: 39403
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.9.107.60"
      port: 46817
    }
    bound_http_address {
      host: "127.9.107.60"
      port: 42735
    }
  }
}

21:04:33.540 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

21:04:33.543 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.9.107.1"
      port: 34711
    }
    bound_http_address {
      host: "127.9.107.1"
      port: 44535
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.9.107.2"
      port: 42877
    }
    bound_http_address {
      host: "127.9.107.2"
      port: 39083
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.9.107.3"
      port: 42267
    }
    bound_http_address {
      host: "127.9.107.3"
      port: 38725
    }
  }
}

21:04:33.543 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:04:33.711 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
21:04:33.779 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
21:04:33.957 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.107.62:44527(127.9.107.62:44527)] connecting to peer
21:04:34.009 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.107.60:46817(127.9.107.60:46817)] connecting to peer
21:04:34.012 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.107.61:39893(127.9.107.61:39893)] connecting to peer
21:04:34.036 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.9.107.61:39893(127.9.107.61:39893)] Successfully connected to peer
21:04:34.037 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.107.62:44527(127.9.107.62:44527)] Successfully connected to peer
21:04:34.043 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.9.107.60:46817(127.9.107.60:46817)] Successfully connected to peer
21:04:34.162 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.9.107.60
21:04:34.162 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.9.107.62
21:04:34.165 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.9.107.61
21:04:34.417 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xf6c07d30, L:/127.0.0.1:51876 - R:/127.9.107.61:39893] using SASL/PLAIN
21:04:34.417 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xe9857ae6, L:/127.0.0.1:50996 - R:/127.9.107.60:46817] using SASL/PLAIN
21:04:34.417 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x50ae465d, L:/127.0.0.1:54004 - R:/127.9.107.62:44527] using SASL/PLAIN
21:04:34.458 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:04:34.459 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=537), Traces: [0ms] refreshing cache from master, [23ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.107.62:44527, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.107.60:46817, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.107.61:39893, [509ms] Sub RPC ConnectToMaster: received response from server master-127.9.107.61:39893: OK, [509ms] Sub RPC ConnectToMaster: received response from server master-127.9.107.62:44527: OK, [510ms] Sub RPC ConnectToMaster: received response from server master-127.9.107.60:46817: OK, deferred=null) after lookup
21:04:34.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.499053 10359 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:04:34.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.509317  9844 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50996:
21:04:34.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909"
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:04:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
21:04:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:34.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.513490  9844 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:34.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table c7d74045fdbf466a8c96c6f580f75c3a
21:04:34.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.603322 10012 tablet_service.cc:1511] Processing CreateTablet for tablet a4634c0115174665ab84366629e41c73 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909 [id=c7d74045fdbf466a8c96c6f580f75c3a]), partition=RANGE (key) PARTITION UNBOUNDED
21:04:34.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.605458 10012 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a4634c0115174665ab84366629e41c73. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:34.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.630920 10294 tablet_service.cc:1511] Processing CreateTablet for tablet a4634c0115174665ab84366629e41c73 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909 [id=c7d74045fdbf466a8c96c6f580f75c3a]), partition=RANGE (key) PARTITION UNBOUNDED
21:04:34.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.632741 10294 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a4634c0115174665ab84366629e41c73. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:34.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.635804 10395 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Bootstrap starting.
21:04:34.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.641331 10395 tablet_bootstrap.cc:654] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Neither blocks nor log segments found. Creating new log.
21:04:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.641326 10153 tablet_service.cc:1511] Processing CreateTablet for tablet a4634c0115174665ab84366629e41c73 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909 [id=c7d74045fdbf466a8c96c6f580f75c3a]), partition=RANGE (key) PARTITION UNBOUNDED
21:04:34.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.643693 10153 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a4634c0115174665ab84366629e41c73. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.644302 10395 log.cc:826] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Log is configured to *not* fsync() on all Append() calls
21:04:34.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.650833 10395 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: No bootstrap required, opened a new log
21:04:34.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.651292 10395 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Time spent bootstrapping tablet: real 0.017s	user 0.008s	sys 0.004s
21:04:34.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.660801 10397 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Bootstrap starting.
21:04:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.670152 10397 tablet_bootstrap.cc:654] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Neither blocks nor log segments found. Creating new log.
21:04:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.673086 10397 log.cc:826] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Log is configured to *not* fsync() on all Append() calls
21:04:34.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.693087 10395 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:34.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.696416 10398 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap starting.
21:04:34.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.694118 10395 raft_consensus.cc:385] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.697093 10395 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aa68dbc27a334b3bb725578a111601e4, State: Initialized, Role: FOLLOWER
21:04:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.698462 10397 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: No bootstrap required, opened a new log
21:04:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.698343 10395 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.699076 10397 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Time spent bootstrapping tablet: real 0.039s	user 0.022s	sys 0.011s
21:04:34.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.707923 10398 tablet_bootstrap.cc:654] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Neither blocks nor log segments found. Creating new log.
21:04:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.711258 10398 log.cc:826] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Log is configured to *not* fsync() on all Append() calls
21:04:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.724833 10395 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Time spent starting tablet: real 0.073s	user 0.034s	sys 0.021s
21:04:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.726747 10398 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: No bootstrap required, opened a new log
21:04:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.727558 10398 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Time spent bootstrapping tablet: real 0.032s	user 0.013s	sys 0.010s
21:04:34.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.737969 10397 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:34.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.739382 10397 raft_consensus.cc:385] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.739687 10397 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95d79f47ce794c54ab09ac6a3908316d, State: Initialized, Role: FOLLOWER
21:04:34.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.740664 10397 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:34.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.744930 10397 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Time spent starting tablet: real 0.046s	user 0.036s	sys 0.000s
21:04:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.757321 10398 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.758028 10398 raft_consensus.cc:385] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.758313 10398 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 18c80cf9fb014eb9bc13a9e176e05a00, State: Initialized, Role: FOLLOWER
21:04:34.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.759148 10398 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.762066 10398 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Time spent starting tablet: real 0.034s	user 0.032s	sys 0.001s
21:04:34.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.805739 10403 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.806468 10403 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.809329 10403 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.809291 10087 tablet.cc:2406] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:34.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.825538 10032 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aa68dbc27a334b3bb725578a111601e4" is_pre_election: true
21:04:34.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.826404 10032 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 18c80cf9fb014eb9bc13a9e176e05a00 in term 0.
21:04:34.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.826321 10314 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95d79f47ce794c54ab09ac6a3908316d" is_pre_election: true
21:04:34.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.827121 10314 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 18c80cf9fb014eb9bc13a9e176e05a00 in term 0.
21:04:34.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.827788 10108 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, aa68dbc27a334b3bb725578a111601e4; no voters: 
21:04:34.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.828557 10403 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:34.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.828890 10403 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:34.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.829182 10403 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 0 FOLLOWER]: Advancing to term 1
21:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.833869 10403 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.835439 10403 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 1 election: Requested vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.836635 10032 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aa68dbc27a334b3bb725578a111601e4"
21:04:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.837080 10314 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:04:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.837159 10032 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 0 FOLLOWER]: Advancing to term 1
21:04:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.837525 10314 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 0 FOLLOWER]: Advancing to term 1
21:04:34.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.841612 10032 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 18c80cf9fb014eb9bc13a9e176e05a00 in term 1.
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.841950 10314 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 18c80cf9fb014eb9bc13a9e176e05a00 in term 1.
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.842734 10108 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, aa68dbc27a334b3bb725578a111601e4; no voters: 
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.843462 10403 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 1 FOLLOWER]: Leader election won for term 1
21:04:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.845480 10403 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 1 LEADER]: Becoming Leader. State: Replica: 18c80cf9fb014eb9bc13a9e176e05a00, State: Running, Role: LEADER
21:04:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.846343 10403 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:34.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.860013  9844 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 reported cstate change: term changed from 0 to 1, leader changed from <none> to 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2). New cstate: current_term: 1 leader_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: UNKNOWN } } }
21:04:34.890 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table c7d74045fdbf466a8c96c6f580f75c3a
21:04:34.910 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:34.915 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299995, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:34.917 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [11ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=null) after lookup
21:04:34.926 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:34.928 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] Successfully connected to peer
21:04:34.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.935177 10221 tablet.cc:2406] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.963459 10367 tablet.cc:2406] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:34.999 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x66725ee0, L:/127.0.0.1:45400 - R:/127.9.107.2:42877] using TOKEN/null
21:04:35.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:35.014813 10314 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 1 FOLLOWER]: Refusing update from remote peer 18c80cf9fb014eb9bc13a9e176e05a00: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:35.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:35.014796 10032 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 1 FOLLOWER]: Refusing update from remote peer 18c80cf9fb014eb9bc13a9e176e05a00: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:35.016719 10406 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:04:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:35.017922 10403 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [LEADER]: Connected to new peer: Peer: permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:35.041842 10410 mvcc.cc:204] Tried to move back new op lower bound from 7305398988842799104 to 7305398988177805312. Current Snapshot: MvccSnapshot[applied={T|T < 7305398988842799104}]
21:04:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:35.119514 10413 mvcc.cc:204] Tried to move back new op lower bound from 7305398988842799104 to 7305398988177805312. Current Snapshot: MvccSnapshot[applied={T|T < 7305398988842799104}]
21:04:35.310 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:35.318 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.107.2:42877
21:04:35.321 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:35.326676  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 10081
21:04:35.366 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:35.367 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] handling channelInactive
21:04:35.368 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state READY due to: connection closed
21:04:35.368 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: connection closed
21:04:35.386 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:35.387 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:35.387 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [20ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@2045612860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@67d786e4 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ec5c4bd -> wakeup thread Time-limited test)) after lookup
21:04:35.388 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:35.391 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.391 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.391 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.407 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:35.408 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:35.408 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [20ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [22ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [25ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [41ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@2045612860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@67d786e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ec5c4bd -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953))) after lookup
21:04:35.409 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:35.411 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.411 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.412 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.428 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:35.428 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:35.429 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [20ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [22ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [25ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [41ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [43ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [46ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [46ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [62ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@2045612860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@67d786e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953) -> (continuation of Deferred@809765111 after retry RPC@574737115), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ec5c4bd -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953) -> (continuation of Deferred@809765111 after retry RPC@574737115))) after lookup
21:04:35.430 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:35.432 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.432 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.433 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.486 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:35.487 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:35.487 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=120), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [20ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [22ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [25ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [41ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [43ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [46ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [46ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [62ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [64ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [67ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [67ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [120ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@2045612860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@67d786e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953) -> (continuation of Deferred@809765111 after retry RPC@574737115) -> (continuation of Deferred@1114580896 after retry RPC@617692055), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ec5c4bd -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953) -> (continuation of Deferred@809765111 after retry RPC@574737115) -> (continuation of Deferred@1114580896 after retry RPC@617692055))) after lookup
21:04:35.488 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:35.489 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.489 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.490 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.949 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:35.949 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:35.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=582), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [20ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [22ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [25ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [41ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [43ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [46ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [46ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [62ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [64ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [67ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [67ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [120ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [121ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [123ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [124ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [576ms] refreshing cache from master, [577ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [582ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@2045612860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@67d786e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953) -> (continuation of Deferred@809765111 after retry RPC@574737115) -> (continuation of Deferred@1114580896 after retry RPC@617692055) -> (continuation of Deferred@1071836401 after retry RPC@901228109), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ec5c4bd -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953) -> (continuation of Deferred@809765111 after retry RPC@574737115) -> (continuation of Deferred@1114580896 after retry RPC@617692055) -> (continuation of Deferred@1071836401 after retry RPC@901228109))) after lookup
21:04:35.951 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:35.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:35.952 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:36.027 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:36.027 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:36.028 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=661), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [20ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [22ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [25ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [41ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [43ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [46ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [46ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [62ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [64ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [67ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [67ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [120ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [121ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [123ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [124ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [576ms] refreshing cache from master, [577ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [582ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [584ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [586ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [586ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [656ms] refreshing cache from master, [656ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [660ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@2045612860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@67d786e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953) -> (continuation of Deferred@809765111 after retry RPC@574737115) -> (continuation of Deferred@1114580896 after retry RPC@617692055) -> (continuation of Deferred@1071836401 after retry RPC@901228109) -> (continuation of Deferred@1837554964 after retry RPC@1703643943), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ec5c4bd -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953) -> (continuation of Deferred@809765111 after retry RPC@574737115) -> (continuation of Deferred@1114580896 after retry RPC@617692055) -> (continuation of Deferred@1071836401 after retry RPC@901228109) -> (continuation of Deferred@1837554964 after retry RPC@1703643943))) after lookup
21:04:36.029 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:36.030 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:36.030 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:36.030 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:36.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.694665 10425 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 18c80cf9fb014eb9bc13a9e176e05a00)
21:04:36.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.695261 10425 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:36.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.698278 10425 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:36.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.702767  9965 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:04:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.707360  9965 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:04:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.710373 10430 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 18c80cf9fb014eb9bc13a9e176e05a00)
21:04:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.710861 10430 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:36.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.712848 10314 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "aa68dbc27a334b3bb725578a111601e4" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "95d79f47ce794c54ab09ac6a3908316d" is_pre_election: true
21:04:36.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.713340 10314 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aa68dbc27a334b3bb725578a111601e4 in term 1.
21:04:36.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.714414 10430 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:36.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.716241  9966 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 95d79f47ce794c54ab09ac6a3908316d, aa68dbc27a334b3bb725578a111601e4; no voters: 18c80cf9fb014eb9bc13a9e176e05a00
21:04:36.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.717165 10247 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:04:36.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.717214 10425 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:04:36.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.717559 10425 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 18c80cf9fb014eb9bc13a9e176e05a00)
21:04:36.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.717916 10425 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 1 FOLLOWER]: Advancing to term 2
21:04:36.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.722982 10247 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:04:36.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.724588 10425 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.727326 10425 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 2 election: Requested vote from peers 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.728402 10314 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "aa68dbc27a334b3bb725578a111601e4" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.728945 10314 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 1 FOLLOWER]: Advancing to term 2
21:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.728672 10032 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "95d79f47ce794c54ab09ac6a3908316d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "aa68dbc27a334b3bb725578a111601e4" is_pre_election: true
21:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.729617 10032 raft_consensus.cc:2393] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 95d79f47ce794c54ab09ac6a3908316d in current term 2: Already voted for candidate aa68dbc27a334b3bb725578a111601e4 in this term.
21:04:36.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.730989  9965 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:04:36.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.731128 10249 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 95d79f47ce794c54ab09ac6a3908316d; no voters: 18c80cf9fb014eb9bc13a9e176e05a00, aa68dbc27a334b3bb725578a111601e4
21:04:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.734340 10314 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aa68dbc27a334b3bb725578a111601e4 in term 2.
21:04:36.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.734733 10430 raft_consensus.cc:2749] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:04:36.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.735262  9966 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 95d79f47ce794c54ab09ac6a3908316d, aa68dbc27a334b3bb725578a111601e4; no voters: 18c80cf9fb014eb9bc13a9e176e05a00
21:04:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.735860 10425 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 2 FOLLOWER]: Leader election won for term 2
21:04:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.737632 10425 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 2 LEADER]: Becoming Leader. State: Replica: aa68dbc27a334b3bb725578a111601e4, State: Running, Role: LEADER
21:04:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.738652 10425 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.748545  9844 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 reported cstate change: term changed from 1 to 2, leader changed from 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2) to aa68dbc27a334b3bb725578a111601e4 (127.9.107.1). New cstate: current_term: 2 leader_uuid: "aa68dbc27a334b3bb725578a111601e4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: UNKNOWN } } }
21:04:36.807 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:36.807 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:36.808 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1441), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [20ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [22ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [25ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [41ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [43ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [46ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [46ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [62ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [64ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [67ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [67ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [120ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [121ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [123ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [124ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [576ms] refreshing cache from master, [577ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [582ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [584ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [586ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [586ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [656ms] refreshing cache from master, [656ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [660ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [663ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [664ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [664ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [1436ms] refreshing cache from master, [1436ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1440ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@2045612860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@67d786e4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953) -> (continuation of Deferred@809765111 after retry RPC@574737115) -> (continuation of Deferred@1114580896 after retry RPC@617692055) -> (continuation of Deferred@1071836401 after retry RPC@901228109) -> (continuation of Deferred@1837554964 after retry RPC@1703643943) -> (continuation of Deferred@1428290011 after retry RPC@471395826), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ec5c4bd -> wakeup thread Time-limited test -> (continuation of Deferred@1821921054 after retry RPC@1750330953) -> (continuation of Deferred@809765111 after retry RPC@574737115) -> (continuation of Deferred@1114580896 after retry RPC@617692055) -> (continuation of Deferred@1071836401 after retry RPC@901228109) -> (continuation of Deferred@1837554964 after retry RPC@1703643943) -> (continuation of Deferred@1428290011 after retry RPC@471395826))) after lookup
21:04:36.809 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:36.810 [DEBUG - kudu-nio-2] (Connection.java:757) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] Successfully connected to peer
21:04:36.837 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x3232b3df, L:/127.0.0.1:58232 - R:/127.9.107.1:34711] using TOKEN/null
21:04:36.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.847533 10314 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 2 FOLLOWER]: Refusing update from remote peer aa68dbc27a334b3bb725578a111601e4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
21:04:36.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.849488 10435 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
21:04:36.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.850400  9965 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 -> Peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Couldn't send request to peer 18c80cf9fb014eb9bc13a9e176e05a00. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:36.904 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.107.2:42877
21:04:36.905 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.906944  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/logs
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.2:42877
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.2
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39083
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.2
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:37.226914 10454 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:37.227370 10454 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:37.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:37.227818 10454 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:37.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:37.260490 10454 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:37.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:37.261497 10454 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.2
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:37.298967 10454 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.2:42877
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.2
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39083
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.2
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/logs
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:37.300448 10454 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:37.302054 10454 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:37.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:37.314749 10460 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:38.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:37.316512 10461 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.532706 10454 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.218s	user 0.608s	sys 0.607s
21:04:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.533072 10454 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.219s	user 0.608s	sys 0.607s
21:04:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.533293 10454 server_base.cc:1061] running on GCE node
21:04:38.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.533738 10463 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:38.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.534888 10454 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:38.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.537259 10454 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:38.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.538676 10454 hybrid_clock.cc:648] HybridClock initialized: now 1783544678538630 us; error 57 us; skew 500 ppm
21:04:38.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.539507 10454 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:38.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.547153 10454 webserver.cc:534] Webserver started at http://127.9.107.2:39083/ using document root <none> and password file <none>
21:04:38.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.548134 10454 fs_manager.cc:362] Metadata directory not provided
21:04:38.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.548369 10454 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.557171 10454 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.000s
21:04:38.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.562458 10472 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.563854 10454 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.005s	sys 0.000s
21:04:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.564195 10454 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18c80cf9fb014eb9bc13a9e176e05a00"
21:04:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:30 on dist-test-slave-qc6t"
21:04:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.566339 10454 fs_report.cc:389] FS layout report
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/data
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.632905 10454 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.634493 10454 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:38.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.635016 10454 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:38.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.637598 10454 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:38.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.643158 10479 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:04:38.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.651526 10454 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.651795 10454 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
21:04:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.652108 10454 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.657213 10454 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.657454 10454 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.006s	sys 0.000s
21:04:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.657963 10479 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap starting.
21:04:38.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.711265 10479 log.cc:826] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Log is configured to *not* fsync() on all Append() calls
21:04:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.738983 10479 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:38.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.740891 10479 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap complete.
21:04:38.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.742646 10479 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Time spent bootstrapping tablet: real 0.085s	user 0.052s	sys 0.029s
21:04:38.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.763211 10479 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:38.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.766137 10479 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 18c80cf9fb014eb9bc13a9e176e05a00, State: Initialized, Role: FOLLOWER
21:04:38.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.767333 10479 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.773120 10479 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Time spent starting tablet: real 0.030s	user 0.029s	sys 0.000s
21:04:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.821130 10454 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.2:42877
21:04:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.821511 10587 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.2:42877 every 8 connection(s)
21:04:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.823966 10454 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:04:38.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.825137  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 10454
21:04:38.826 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:38.855 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:38.856 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.107.1:34711
21:04:38.857 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:04:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.857648  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 9925
21:04:38.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.886360 10589 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:04:38.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.886857 10589 heartbeater.cc:461] Registering TS with master...
21:04:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.888219 10588 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:04:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.888187 10589 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:04:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.888578 10588 heartbeater.cc:461] Registering TS with master...
21:04:38.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.889427 10588 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:04:38.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.892767  9844 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:38.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.892990  9764 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:38.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.896188  9844 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.2:45975
21:04:38.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.898454 10590 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:04:38.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.898790 10590 heartbeater.cc:461] Registering TS with master...
21:04:38.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.899780 10590 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:04:38.904 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:38.904 [DEBUG - kudu-nio-2] (Connection.java:239) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] handling channelInactive
21:04:38.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.902998  9689 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:38.905 [DEBUG - kudu-nio-2] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state READY due to: connection closed
21:04:38.905 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: connection closed
21:04:38.927 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:38.927 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:38.927 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [1ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [22ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@614557062(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7f0d7b9d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d9bcdcd -> wakeup thread Time-limited test)) after lookup
21:04:38.928 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:38.930 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:38.930 [DEBUG - kudu-nio-3] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:38.930 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:38.946 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:38.947 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:38.947 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [1ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [22ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [24ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [26ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [26ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@614557062(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7f0d7b9d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1293300285 after retry RPC@1049656311), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d9bcdcd -> wakeup thread Time-limited test -> (continuation of Deferred@1293300285 after retry RPC@1049656311))) after lookup
21:04:38.948 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:38.949 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:38.949 [DEBUG - kudu-nio-4] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:38.949 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:38.986 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:38.986 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:38.987 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=82), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [1ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [22ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [24ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [26ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [26ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [43ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [45ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [45ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [82ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@614557062(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7f0d7b9d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1293300285 after retry RPC@1049656311) -> (continuation of Deferred@2067797288 after retry RPC@1177720436), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d9bcdcd -> wakeup thread Time-limited test -> (continuation of Deferred@1293300285 after retry RPC@1049656311) -> (continuation of Deferred@2067797288 after retry RPC@1177720436))) after lookup
21:04:38.987 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:38.988 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:38.989 [DEBUG - kudu-nio-5] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:38.989 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:39.026 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:39.026 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:39.027 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=122), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [1ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [22ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [24ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [26ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [26ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [43ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [45ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [45ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [82ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [83ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [85ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [85ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [118ms] refreshing cache from master, [118ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [122ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@614557062(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7f0d7b9d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1293300285 after retry RPC@1049656311) -> (continuation of Deferred@2067797288 after retry RPC@1177720436) -> (continuation of Deferred@845313135 after retry RPC@1613874529), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d9bcdcd -> wakeup thread Time-limited test -> (continuation of Deferred@1293300285 after retry RPC@1049656311) -> (continuation of Deferred@2067797288 after retry RPC@1177720436) -> (continuation of Deferred@845313135 after retry RPC@1613874529))) after lookup
21:04:39.028 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:39.029 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:39.029 [DEBUG - kudu-nio-6] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:39.029 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:39.506 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:39.506 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:39.507 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=602), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [1ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [22ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [24ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [26ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [26ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [43ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [45ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [45ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [82ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [83ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [85ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [85ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [118ms] refreshing cache from master, [118ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [122ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [123ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [125ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [125ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [598ms] refreshing cache from master, [598ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [602ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@614557062(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7f0d7b9d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1293300285 after retry RPC@1049656311) -> (continuation of Deferred@2067797288 after retry RPC@1177720436) -> (continuation of Deferred@845313135 after retry RPC@1613874529) -> (continuation of Deferred@655432696 after retry RPC@1231407286), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d9bcdcd -> wakeup thread Time-limited test -> (continuation of Deferred@1293300285 after retry RPC@1049656311) -> (continuation of Deferred@2067797288 after retry RPC@1177720436) -> (continuation of Deferred@845313135 after retry RPC@1613874529) -> (continuation of Deferred@655432696 after retry RPC@1231407286))) after lookup
21:04:39.508 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:39.509 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:39.509 [DEBUG - kudu-nio-7] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:39.510 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:39.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.852432 10600 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 2 FOLLOWER]: Starting pre-election (detected failure of leader aa68dbc27a334b3bb725578a111601e4)
21:04:39.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.852802 10600 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:39.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.854425 10600 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:39.861032 10249 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711): Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111)
21:04:39.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.867290 10542 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "95d79f47ce794c54ab09ac6a3908316d" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" is_pre_election: true
21:04:39.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.868032 10542 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 95d79f47ce794c54ab09ac6a3908316d in term 1.
21:04:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.869421 10247 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, 95d79f47ce794c54ab09ac6a3908316d; no voters: aa68dbc27a334b3bb725578a111601e4
21:04:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.870040 10600 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 2 FOLLOWER]: Leader pre-election won for term 3
21:04:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.870306 10600 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 2 FOLLOWER]: Starting leader election (detected failure of leader aa68dbc27a334b3bb725578a111601e4)
21:04:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.870613 10600 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 2 FOLLOWER]: Advancing to term 3
21:04:39.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.874826 10600 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:39.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.876400 10600 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 3 election: Requested vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:39.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.877396 10542 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "95d79f47ce794c54ab09ac6a3908316d" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "18c80cf9fb014eb9bc13a9e176e05a00"
21:04:39.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.877851 10542 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 1 FOLLOWER]: Advancing to term 3
21:04:39.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:39.879549 10249 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711): Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111)
21:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.883896 10542 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 95d79f47ce794c54ab09ac6a3908316d in term 3.
21:04:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.884835 10247 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, 95d79f47ce794c54ab09ac6a3908316d; no voters: aa68dbc27a334b3bb725578a111601e4
21:04:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.885468 10600 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 3 FOLLOWER]: Leader election won for term 3
21:04:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.886500 10600 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 3 LEADER]: Becoming Leader. State: Replica: 95d79f47ce794c54ab09ac6a3908316d, State: Running, Role: LEADER
21:04:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.887573 10600 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:39.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.895651  9843 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d reported cstate change: term changed from 2 to 3, leader changed from aa68dbc27a334b3bb725578a111601e4 (127.9.107.1) to 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3). New cstate: current_term: 3 leader_uuid: "95d79f47ce794c54ab09ac6a3908316d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: HEALTHY } } }
21:04:39.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:39.900202 10588 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:04:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:40.370621 10249 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d -> Peer aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711): Couldn't send request to peer aa68dbc27a334b3bb725578a111601e4. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:40.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:40.497498 10542 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 3 FOLLOWER]: Refusing update from remote peer 95d79f47ce794c54ab09ac6a3908316d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
21:04:40.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:40.498844 10600 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [LEADER]: Connected to new peer: Peer: permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
21:04:40.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:40.587 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:40.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1683), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [1ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [18ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [22ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [24ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [26ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [26ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [43ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [45ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [45ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [82ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [83ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [85ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [85ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [118ms] refreshing cache from master, [118ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [122ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [123ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [125ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [125ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [598ms] refreshing cache from master, [598ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [602ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [603ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [605ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [606ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [1678ms] refreshing cache from master, [1678ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1682ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@614557062(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7f0d7b9d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1293300285 after retry RPC@1049656311) -> (continuation of Deferred@2067797288 after retry RPC@1177720436) -> (continuation of Deferred@845313135 after retry RPC@1613874529) -> (continuation of Deferred@655432696 after retry RPC@1231407286) -> (continuation of Deferred@90676961 after retry RPC@692732874), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d9bcdcd -> wakeup thread Time-limited test -> (continuation of Deferred@1293300285 after retry RPC@1049656311) -> (continuation of Deferred@2067797288 after retry RPC@1177720436) -> (continuation of Deferred@845313135 after retry RPC@1613874529) -> (continuation of Deferred@655432696 after retry RPC@1231407286) -> (continuation of Deferred@90676961 after retry RPC@692732874))) after lookup
21:04:40.588 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:40.589 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] Successfully connected to peer
21:04:40.613 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa098b79e, L:/127.0.0.1:49624 - R:/127.9.107.3:42267] using TOKEN/null
21:04:40.664 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.107.1:34711
21:04:40.665 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:40.667344  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data
21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/logs
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.1:34711
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.1
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44535
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.1
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:40.969375 10631 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:40.969846 10631 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:40.970352 10631 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:41.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:41.004299 10631 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:41.005255 10631 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.1
21:04:41.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:41.042511 10631 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.1:34711
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.1
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44535
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.1
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/logs
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:41.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:41.043850 10631 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:41.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:41.045686 10631 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:41.057967 10637 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:42.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:42.096396 10249 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111) [suppressed 6 similar messages]
21:04:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:41.059940 10638 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:41.062130 10640 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:42.233556 10639 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1169 milliseconds
21:04:42.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.233697 10631 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:04:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.235150 10631 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.237339 10631 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:42.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.238840 10631 hybrid_clock.cc:648] HybridClock initialized: now 1783544682238774 us; error 80 us; skew 500 ppm
21:04:42.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.239735 10631 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.245779 10631 webserver.cc:534] Webserver started at http://127.9.107.1:44535/ using document root <none> and password file <none>
21:04:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.246707 10631 fs_manager.cc:362] Metadata directory not provided
21:04:42.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.246929 10631 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.255059 10631 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.004s	sys 0.000s
21:04:42.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.259646 10649 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:42.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.260730 10631 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.004s
21:04:42.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.261077 10631 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:42.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa68dbc27a334b3bb725578a111601e4"
21:04:42.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:28 on dist-test-slave-qc6t"
21:04:42.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.263299 10631 fs_report.cc:389] FS layout report
21:04:42.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:42.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/data
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.313606 10631 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.315259 10631 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.315764 10631 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.318601 10631 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:42.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.324927 10656 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:04:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.336862 10631 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.337124 10631 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.014s	user 0.002s	sys 0.001s
21:04:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.337458 10631 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:42.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.342125 10631 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:42.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.342319 10631 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
21:04:42.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.342931 10656 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Bootstrap starting.
21:04:42.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.395569 10656 log.cc:826] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Log is configured to *not* fsync() on all Append() calls
21:04:42.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.436646 10656 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:42.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.437654 10656 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Bootstrap complete.
21:04:42.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.439292 10656 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Time spent bootstrapping tablet: real 0.097s	user 0.068s	sys 0.025s
21:04:42.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.460049 10656 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:42.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.463637 10656 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: aa68dbc27a334b3bb725578a111601e4, State: Initialized, Role: FOLLOWER
21:04:42.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.465549 10656 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.470046 10656 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Time spent starting tablet: real 0.030s	user 0.031s	sys 0.000s
21:04:42.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:42.491374 10249 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d -> Peer aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711): Couldn't send request to peer aa68dbc27a334b3bb725578a111601e4. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:04:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.506739 10631 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.1:34711
21:04:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.507043 10764 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.1:34711 every 8 connection(s)
21:04:42.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.509140 10631 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:04:42.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.519269  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 10631
21:04:42.520 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:42.551 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:42.552 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.107.3:42267
21:04:42.552 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:04:42.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.553229  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 10228
21:04:42.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.557132 10767 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:04:42.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.557581 10767 heartbeater.cc:461] Registering TS with master...
21:04:42.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.558921 10767 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:04:42.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.562947 10766 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:04:42.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.563282 10766 heartbeater.cc:461] Registering TS with master...
21:04:42.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.563397  9688 ts_manager.cc:194] Re-registered known tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:04:42.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.564198 10766 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:04:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.567874  9764 ts_manager.cc:194] Re-registered known tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:04:42.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.571012 10765 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:04:42.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.571311 10765 heartbeater.cc:461] Registering TS with master...
21:04:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.572057 10765 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:04:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.574400  9843 ts_manager.cc:194] Re-registered known tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:04:42.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:42.576432  9843 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.1:54591
21:04:42.581 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] handling channelInactive
21:04:42.581 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state READY due to: connection closed
21:04:42.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:42.581179  9829 connection.cc:573] server connection from 127.9.107.3:58219 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:42.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:42.582 [DEBUG - Time-limited test] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:42.583 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.584 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.584 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.606 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:42.606 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:42.607 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1989158103(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@6b1b4787 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f7128 -> wakeup thread Time-limited test)) after lookup
21:04:42.607 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:42.608 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.608 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.608 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.626 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:42.626 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:42.627 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [26ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [26ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1989158103(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@6b1b4787 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@747723797 after retry RPC@1349250718), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f7128 -> wakeup thread Time-limited test -> (continuation of Deferred@747723797 after retry RPC@1349250718))) after lookup
21:04:42.627 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:42.628 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.628 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.629 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.667 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:42.667 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:42.667 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=85), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [26ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [26ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [45ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [46ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [47ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [85ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1989158103(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@6b1b4787 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@747723797 after retry RPC@1349250718) -> (continuation of Deferred@138097645 after retry RPC@120271722), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f7128 -> wakeup thread Time-limited test -> (continuation of Deferred@747723797 after retry RPC@1349250718) -> (continuation of Deferred@138097645 after retry RPC@120271722))) after lookup
21:04:42.668 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:42.669 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.669 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.669 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.727 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:42.727 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:42.728 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=145), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [26ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [26ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [45ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [46ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [47ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [85ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [86ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [87ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [87ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [140ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [144ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1989158103(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@6b1b4787 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@747723797 after retry RPC@1349250718) -> (continuation of Deferred@138097645 after retry RPC@120271722) -> (continuation of Deferred@893351089 after retry RPC@1808904479), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f7128 -> wakeup thread Time-limited test -> (continuation of Deferred@747723797 after retry RPC@1349250718) -> (continuation of Deferred@138097645 after retry RPC@120271722) -> (continuation of Deferred@893351089 after retry RPC@1808904479))) after lookup
21:04:42.728 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:42.730 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.730 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.730 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.806 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:42.807 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:42.807 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=225), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [26ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [26ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [45ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [46ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [47ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [85ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [86ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [87ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [87ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [140ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [144ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [146ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [148ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [148ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [220ms] refreshing cache from master, [221ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [224ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1989158103(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@6b1b4787 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@747723797 after retry RPC@1349250718) -> (continuation of Deferred@138097645 after retry RPC@120271722) -> (continuation of Deferred@893351089 after retry RPC@1808904479) -> (continuation of Deferred@542767448 after retry RPC@1181154565), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f7128 -> wakeup thread Time-limited test -> (continuation of Deferred@747723797 after retry RPC@1349250718) -> (continuation of Deferred@138097645 after retry RPC@120271722) -> (continuation of Deferred@893351089 after retry RPC@1808904479) -> (continuation of Deferred@542767448 after retry RPC@1181154565))) after lookup
21:04:42.808 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:42.809 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.809 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:42.809 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:43.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:43.579528 10765 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:04:44.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.026252 10777 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 95d79f47ce794c54ab09ac6a3908316d)
21:04:44.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.026643 10777 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.029019 10777 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:44.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.032513 10474 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111)
21:04:44.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.036295 10474 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111)
21:04:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.041010 10719 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "aa68dbc27a334b3bb725578a111601e4" is_pre_election: true
21:04:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.041829 10719 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 18c80cf9fb014eb9bc13a9e176e05a00 in term 2.
21:04:44.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.043156 10475 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, aa68dbc27a334b3bb725578a111601e4; no voters: 95d79f47ce794c54ab09ac6a3908316d
21:04:44.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.043865 10777 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 3 FOLLOWER]: Leader pre-election won for term 4
21:04:44.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.044185 10777 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 95d79f47ce794c54ab09ac6a3908316d)
21:04:44.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.044497 10777 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 3 FOLLOWER]: Advancing to term 4
21:04:44.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.048822 10777 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:44.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.050455 10777 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 4 election: Requested vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.051172 10719 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "aa68dbc27a334b3bb725578a111601e4"
21:04:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.051599 10719 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 2 FOLLOWER]: Advancing to term 4
21:04:44.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.053695 10474 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111)
21:04:44.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.057863 10719 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 18c80cf9fb014eb9bc13a9e176e05a00 in term 4.
21:04:44.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.058746 10475 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, aa68dbc27a334b3bb725578a111601e4; no voters: 95d79f47ce794c54ab09ac6a3908316d
21:04:44.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.059545 10777 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 4 FOLLOWER]: Leader election won for term 4
21:04:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.061189 10777 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 4 LEADER]: Becoming Leader. State: Replica: 18c80cf9fb014eb9bc13a9e176e05a00, State: Running, Role: LEADER
21:04:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.062593 10777 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:44.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.073460  9843 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 reported cstate change: term changed from 3 to 4, leader changed from 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3) to 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2). New cstate: current_term: 4 leader_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: UNKNOWN } } }
21:04:44.286 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:44.286 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:44.286 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1704), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [26ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [26ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [45ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [46ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [47ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [85ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [86ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [87ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [87ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [140ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [144ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [146ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [148ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [148ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [220ms] refreshing cache from master, [221ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [224ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [225ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [227ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [227ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [1700ms] refreshing cache from master, [1700ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1704ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1989158103(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@6b1b4787 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@747723797 after retry RPC@1349250718) -> (continuation of Deferred@138097645 after retry RPC@120271722) -> (continuation of Deferred@893351089 after retry RPC@1808904479) -> (continuation of Deferred@542767448 after retry RPC@1181154565) -> (continuation of Deferred@755848913 after retry RPC@1550981280), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f7128 -> wakeup thread Time-limited test -> (continuation of Deferred@747723797 after retry RPC@1349250718) -> (continuation of Deferred@138097645 after retry RPC@120271722) -> (continuation of Deferred@893351089 after retry RPC@1808904479) -> (continuation of Deferred@542767448 after retry RPC@1181154565) -> (continuation of Deferred@755848913 after retry RPC@1550981280))) after lookup
21:04:44.287 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:44.288 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] Successfully connected to peer
21:04:44.340 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0xf72ad91f, L:/127.0.0.1:38804 - R:/127.9.107.2:42877] using TOKEN/null
21:04:44.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.350862 10719 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 4 FOLLOWER]: Refusing update from remote peer 18c80cf9fb014eb9bc13a9e176e05a00: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
21:04:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.352650 10782 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [LEADER]: Connected to new peer: Peer: permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.001s
21:04:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.353358 10474 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 -> Peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Couldn't send request to peer 95d79f47ce794c54ab09ac6a3908316d. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:44.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.370183 10799 mvcc.cc:204] Tried to move back new op lower bound from 7305399011812274176 to 7305399008827011072. Current Snapshot: MvccSnapshot[applied={T|T < 7305399011812274176}]
21:04:44.418 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.107.3:42267
21:04:44.419 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.420356  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data
21:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/logs
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.3:42267
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.3
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38725
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.3
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.733227 10806 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.733732 10806 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.734177 10806 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:44.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.752372  9875 debug-util.cc:398] Leaking SignalData structure 0x7b08000a6680 after lost signal to thread 9806
21:04:44.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.753060  9875 debug-util.cc:398] Leaking SignalData structure 0x7b0800099700 after lost signal to thread 9878
21:04:44.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.766743 10806 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.767722 10806 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.3
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.806561 10806 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.3:42267
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.3
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38725
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.3
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/logs
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:44.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:44.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:44.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:44.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:44.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:44.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.807987 10806 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:44.810194 10806 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:44.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.828992 10812 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:46.226943 10811 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 10806
21:04:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:46.518136 10811 kernel_stack_watchdog.cc:198] Thread 10806 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 388ms:
21:04:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:04:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:04:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:04:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:04:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:44.843082 10813 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:46.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:46.520401 10815 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:46.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:46.519665 10806 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.683s	user 0.662s	sys 0.902s
21:04:46.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:46.520959 10806 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.685s	user 0.662s	sys 0.902s
21:04:46.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:46.524000 10814 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1687 milliseconds
21:04:46.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.524139 10806 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:04:46.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.525971 10806 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.528873 10806 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.530321 10806 hybrid_clock.cc:648] HybridClock initialized: now 1783544686530135 us; error 191 us; skew 500 ppm
21:04:46.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.531363 10806 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:46.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.539081 10806 webserver.cc:534] Webserver started at http://127.9.107.3:38725/ using document root <none> and password file <none>
21:04:46.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.540279 10806 fs_manager.cc:362] Metadata directory not provided
21:04:46.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.540577 10806 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:46.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.551369 10806 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.005s	sys 0.003s
21:04:46.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.557152 10824 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:46.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.559202 10806 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.001s
21:04:46.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.559669 10806 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:46.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:04:46.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:33 on dist-test-slave-qc6t"
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.562433 10806 fs_report.cc:389] FS layout report
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/data
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:46.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.623188 10806 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.624761 10806 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.625406 10806 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:46.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.628826 10806 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:46.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.635303 10831 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:04:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.643570 10806 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.643797 10806 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
21:04:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.644009 10806 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:46.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.648576 10806 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:46.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.648846 10806 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
21:04:46.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.649307 10831 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Bootstrap starting.
21:04:46.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:46.686175 10474 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 -> Peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Couldn't send request to peer 95d79f47ce794c54ab09ac6a3908316d. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:04:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.719184 10831 log.cc:826] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Log is configured to *not* fsync() on all Append() calls
21:04:46.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.782578 10831 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:46.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.783782 10831 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Bootstrap complete.
21:04:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.785562 10831 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Time spent bootstrapping tablet: real 0.137s	user 0.093s	sys 0.037s
21:04:46.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.813104 10831 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:46.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.816701 10831 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95d79f47ce794c54ab09ac6a3908316d, State: Initialized, Role: FOLLOWER
21:04:46.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.817886 10831 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:46.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.831632 10831 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Time spent starting tablet: real 0.046s	user 0.033s	sys 0.012s
21:04:46.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.851516 10806 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.3:42267
21:04:46.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.851712 10939 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.3:42267 every 8 connection(s)
21:04:46.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.855083 10806 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:04:46.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.864459  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 10806
21:04:46.865 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:46.921 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:46.932 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.107.2:42877
21:04:46.932 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:46.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.932888  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 10454
21:04:46.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.959263 10941 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:04:46.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.959793 10941 heartbeater.cc:461] Registering TS with master...
21:04:46.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.961086 10941 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:04:46.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.962947 10940 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:04:46.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.963300 10940 heartbeater.cc:461] Registering TS with master...
21:04:46.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.964135 10940 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:04:46.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.965801  9763 ts_manager.cc:194] Re-registered known tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:46.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.968209 10942 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:04:46.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.968554 10942 heartbeater.cc:461] Registering TS with master...
21:04:46.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.969002  9843 ts_manager.cc:194] Re-registered known tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:46.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.969476 10942 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:04:46.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.971949  9843 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.3:57271
21:04:46.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:46.972831  9688 ts_manager.cc:194] Re-registered known tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:46.982 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] handling channelInactive
21:04:46.982 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state READY due to: connection closed
21:04:46.982 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:46.983 [DEBUG - Time-limited test] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:46.984 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:46.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:46.985 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.007 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:47.008 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:47.008 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1356268544(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@1cacb1a2 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4711ceeb -> wakeup thread Time-limited test)) after lookup
21:04:47.009 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:47.010 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.010 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.010 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.026 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:47.026 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:47.026 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [27ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1356268544(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@1cacb1a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4711ceeb -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715))) after lookup
21:04:47.027 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:47.028 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.028 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.028 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.066 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:47.066 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:47.066 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=83), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [27ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [44ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [45ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [45ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [82ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1356268544(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@1cacb1a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715) -> (continuation of Deferred@1538292661 after retry RPC@1219319010), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4711ceeb -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715) -> (continuation of Deferred@1538292661 after retry RPC@1219319010))) after lookup
21:04:47.067 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:47.068 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.068 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.068 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.147 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:47.147 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:47.148 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=164), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [27ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [44ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [45ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [45ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [82ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [83ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [85ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [85ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [159ms] refreshing cache from master, [159ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [163ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1356268544(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@1cacb1a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715) -> (continuation of Deferred@1538292661 after retry RPC@1219319010) -> (continuation of Deferred@326729459 after retry RPC@835793466), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4711ceeb -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715) -> (continuation of Deferred@1538292661 after retry RPC@1219319010) -> (continuation of Deferred@326729459 after retry RPC@835793466))) after lookup
21:04:47.148 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:47.150 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.150 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.151 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.206 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:47.206 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:47.207 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=224), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [27ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [44ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [45ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [45ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [82ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [83ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [85ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [85ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [159ms] refreshing cache from master, [159ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [163ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [165ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [168ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [168ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [219ms] refreshing cache from master, [219ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [223ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1356268544(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@1cacb1a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715) -> (continuation of Deferred@1538292661 after retry RPC@1219319010) -> (continuation of Deferred@326729459 after retry RPC@835793466) -> (continuation of Deferred@1276669417 after retry RPC@201804613), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4711ceeb -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715) -> (continuation of Deferred@1538292661 after retry RPC@1219319010) -> (continuation of Deferred@326729459 after retry RPC@835793466) -> (continuation of Deferred@1276669417 after retry RPC@201804613))) after lookup
21:04:47.208 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:47.209 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.209 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.209 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:47.307 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:47.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=324), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [27ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [44ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [45ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [45ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [82ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [83ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [85ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [85ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [159ms] refreshing cache from master, [159ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [163ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [165ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [168ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [168ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [219ms] refreshing cache from master, [219ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [223ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [224ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [226ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [226ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [319ms] refreshing cache from master, [319ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [323ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1356268544(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@1cacb1a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715) -> (continuation of Deferred@1538292661 after retry RPC@1219319010) -> (continuation of Deferred@326729459 after retry RPC@835793466) -> (continuation of Deferred@1276669417 after retry RPC@201804613) -> (continuation of Deferred@747799844 after retry RPC@452574864), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4711ceeb -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715) -> (continuation of Deferred@1538292661 after retry RPC@1219319010) -> (continuation of Deferred@326729459 after retry RPC@835793466) -> (continuation of Deferred@1276669417 after retry RPC@201804613) -> (continuation of Deferred@747799844 after retry RPC@452574864))) after lookup
21:04:47.308 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:47.309 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.309 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.309 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:47.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:47.975761 10940 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:04:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.129300 10951 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 18c80cf9fb014eb9bc13a9e176e05a00)
21:04:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.129715 10951 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:48.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.132201 10951 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:48.145391 10650 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:04:48.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:48.149045 10650 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:04:48.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.155483 10894 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "aa68dbc27a334b3bb725578a111601e4" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "95d79f47ce794c54ab09ac6a3908316d" is_pre_election: true
21:04:48.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.156188 10894 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aa68dbc27a334b3bb725578a111601e4 in term 3.
21:04:48.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.157624 10651 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 95d79f47ce794c54ab09ac6a3908316d, aa68dbc27a334b3bb725578a111601e4; no voters: 18c80cf9fb014eb9bc13a9e176e05a00
21:04:48.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.158325 10951 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 4 FOLLOWER]: Leader pre-election won for term 5
21:04:48.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.158643 10951 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 18c80cf9fb014eb9bc13a9e176e05a00)
21:04:48.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.158871 10951 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 4 FOLLOWER]: Advancing to term 5
21:04:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.163291 10951 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.164852 10951 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 5 election: Requested vote from peers 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.166002 10894 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "aa68dbc27a334b3bb725578a111601e4" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:04:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.166532 10894 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 3 FOLLOWER]: Advancing to term 5
21:04:48.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:48.168529 10650 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:04:48.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.173491 10894 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aa68dbc27a334b3bb725578a111601e4 in term 5.
21:04:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.174649 10651 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 95d79f47ce794c54ab09ac6a3908316d, aa68dbc27a334b3bb725578a111601e4; no voters: 18c80cf9fb014eb9bc13a9e176e05a00
21:04:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.175275 10951 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 5 FOLLOWER]: Leader election won for term 5
21:04:48.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.176959 10951 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 5 LEADER]: Becoming Leader. State: Replica: aa68dbc27a334b3bb725578a111601e4, State: Running, Role: LEADER
21:04:48.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.177881 10951 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:48.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.187840  9843 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 reported cstate change: term changed from 4 to 5, leader changed from 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2) to aa68dbc27a334b3bb725578a111601e4 (127.9.107.1). New cstate: current_term: 5 leader_uuid: "aa68dbc27a334b3bb725578a111601e4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: UNKNOWN } } }
21:04:48.546 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:48.547 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:48.547 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1564), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [27ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [44ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [45ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [45ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [82ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [83ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [85ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [85ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [159ms] refreshing cache from master, [159ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [163ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [165ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [168ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [168ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [219ms] refreshing cache from master, [219ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [223ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [224ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [226ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [226ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [319ms] refreshing cache from master, [319ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [323ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [324ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [326ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [326ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [1559ms] refreshing cache from master, [1559ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1563ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1356268544(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@1cacb1a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715) -> (continuation of Deferred@1538292661 after retry RPC@1219319010) -> (continuation of Deferred@326729459 after retry RPC@835793466) -> (continuation of Deferred@1276669417 after retry RPC@201804613) -> (continuation of Deferred@747799844 after retry RPC@452574864) -> (continuation of Deferred@1486322038 after retry RPC@256725450), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4711ceeb -> wakeup thread Time-limited test -> (continuation of Deferred@380101789 after retry RPC@1142616715) -> (continuation of Deferred@1538292661 after retry RPC@1219319010) -> (continuation of Deferred@326729459 after retry RPC@835793466) -> (continuation of Deferred@1276669417 after retry RPC@201804613) -> (continuation of Deferred@747799844 after retry RPC@452574864) -> (continuation of Deferred@1486322038 after retry RPC@256725450))) after lookup
21:04:48.548 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:48.549 [DEBUG - kudu-nio-7] (Connection.java:757) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] Successfully connected to peer
21:04:48.567 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0xc58f8fea, L:/127.0.0.1:53956 - R:/127.9.107.1:34711] using TOKEN/null
21:04:48.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.577047 10894 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 5 FOLLOWER]: Refusing update from remote peer aa68dbc27a334b3bb725578a111601e4: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 18. (index mismatch)
21:04:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.578740 10956 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.001s
21:04:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:48.579428 10650 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 -> Peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Couldn't send request to peer 18c80cf9fb014eb9bc13a9e176e05a00. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:48.634 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.107.2:42877
21:04:48.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:48.637267  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data
21:04:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/logs
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.2:42877
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.2
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39083
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.2
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:48.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:48.936962 10977 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:48.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:48.937462 10977 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:48.937918 10977 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:48.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:48.971402 10977 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:48.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:48.972245 10977 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.2
21:04:49.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:49.009475 10977 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.2:42877
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.2
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39083
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.2
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/logs
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:49.011164 10977 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:49.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:49.012823 10977 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:49.025641 10983 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:50.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:49.026000 10984 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:50.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:50.192988 10986 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:50.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:50.195822 10985 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1164 milliseconds
21:04:50.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.195926 10977 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:04:50.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.197105 10977 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:50.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.199613 10977 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:50.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.201053 10977 hybrid_clock.cc:648] HybridClock initialized: now 1783544690200977 us; error 82 us; skew 500 ppm
21:04:50.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.201924 10977 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.208352 10977 webserver.cc:534] Webserver started at http://127.9.107.2:39083/ using document root <none> and password file <none>
21:04:50.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.209354 10977 fs_manager.cc:362] Metadata directory not provided
21:04:50.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.209545 10977 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:50.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.219184 10977 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.005s	sys 0.004s
21:04:50.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.225011 10997 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:50.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.226554 10977 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
21:04:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.226962 10977 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18c80cf9fb014eb9bc13a9e176e05a00"
21:04:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:30 on dist-test-slave-qc6t"
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.229645 10977 fs_report.cc:389] FS layout report
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/data
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.287231 10977 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:50.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.288641 10977 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:50.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.289065 10977 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.291625 10977 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:50.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.297261 11004 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:04:50.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.305881 10977 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:50.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.306130 10977 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
21:04:50.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.306344 10977 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.311136 10977 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.311321 10977 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
21:04:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.311836 11004 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap starting.
21:04:50.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.374292 11004 log.cc:826] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Log is configured to *not* fsync() on all Append() calls
21:04:50.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.426841 11004 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:50.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.427686 11004 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap complete.
21:04:50.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.429055 11004 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Time spent bootstrapping tablet: real 0.118s	user 0.078s	sys 0.035s
21:04:50.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.447011 11004 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:50.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.449352 11004 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 18c80cf9fb014eb9bc13a9e176e05a00, State: Initialized, Role: FOLLOWER
21:04:50.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.450595 11004 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:50.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.455598 11004 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Time spent starting tablet: real 0.026s	user 0.018s	sys 0.007s
21:04:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.490746 10977 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.2:42877
21:04:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.490993 11112 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.2:42877 every 8 connection(s)
21:04:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.493095 10977 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:04:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.498284  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 10977
21:04:50.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:50.526 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:50.527 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.107.1:34711
21:04:50.527 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:04:50.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.529331  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 10631
21:04:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.553503 11115 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:04:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.554003 11115 heartbeater.cc:461] Registering TS with master...
21:04:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.554065 11113 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:04:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.554337 11113 heartbeater.cc:461] Registering TS with master...
21:04:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.555269 11113 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:04:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.555462 11115 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:04:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.556169 11114 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:04:50.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.556464 11114 heartbeater.cc:461] Registering TS with master...
21:04:50.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.557353 11114 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:04:50.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.559995  9843 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.560372  9688 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.562250  9763 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:50.562974  9843 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.2:59823
21:04:50.566 [DEBUG - kudu-nio-7] (Connection.java:239) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] handling channelInactive
21:04:50.566 [DEBUG - kudu-nio-7] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state READY due to: connection closed
21:04:50.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:50.567 [DEBUG - Time-limited test] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:50.568 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.569 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.586 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:50.587 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:50.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@90623966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@60c01db9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b6ffffd -> wakeup thread Time-limited test)) after lookup
21:04:50.587 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:50.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.588 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.606 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:50.606 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:50.606 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [20ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [21ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [21ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@90623966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@60c01db9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b6ffffd -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018))) after lookup
21:04:50.607 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:50.608 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.608 [DEBUG - kudu-nio-2] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.608 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.646 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:50.646 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:50.646 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [20ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [21ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [21ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [39ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [41ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [41ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [78ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@90623966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@60c01db9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b6ffffd -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856))) after lookup
21:04:50.647 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:50.648 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.648 [DEBUG - kudu-nio-3] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.648 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.766 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:50.766 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:50.767 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=199), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [20ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [21ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [21ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [39ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [41ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [41ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [78ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [80ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [81ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [81ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [195ms] refreshing cache from master, [195ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [199ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@90623966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@60c01db9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856) -> (continuation of Deferred@1431237978 after retry RPC@1956158774), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b6ffffd -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856) -> (continuation of Deferred@1431237978 after retry RPC@1956158774))) after lookup
21:04:50.768 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:50.769 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.769 [DEBUG - kudu-nio-4] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.769 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.846 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:50.846 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:50.847 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=279), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [20ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [21ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [21ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [39ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [41ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [41ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [78ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [80ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [81ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [81ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [195ms] refreshing cache from master, [195ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [199ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [200ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [202ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [202ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [275ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [279ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@90623966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@60c01db9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856) -> (continuation of Deferred@1431237978 after retry RPC@1956158774) -> (continuation of Deferred@97962348 after retry RPC@890734029), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b6ffffd -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856) -> (continuation of Deferred@1431237978 after retry RPC@1956158774) -> (continuation of Deferred@97962348 after retry RPC@890734029))) after lookup
21:04:50.848 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:50.849 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.849 [DEBUG - kudu-nio-5] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:50.849 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:51.226 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:51.226 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:51.227 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=659), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [20ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [21ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [21ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [39ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [41ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [41ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [78ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [80ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [81ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [81ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [195ms] refreshing cache from master, [195ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [199ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [200ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [202ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [202ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [275ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [279ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [280ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [282ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [282ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [655ms] refreshing cache from master, [655ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [659ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@90623966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@60c01db9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856) -> (continuation of Deferred@1431237978 after retry RPC@1956158774) -> (continuation of Deferred@97962348 after retry RPC@890734029) -> (continuation of Deferred@775737757 after retry RPC@663486586), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b6ffffd -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856) -> (continuation of Deferred@1431237978 after retry RPC@1956158774) -> (continuation of Deferred@97962348 after retry RPC@890734029) -> (continuation of Deferred@775737757 after retry RPC@663486586))) after lookup
21:04:51.227 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:51.228 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:51.228 [DEBUG - kudu-nio-6] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:51.228 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:51.287 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:51.287 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:04:51.288 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=720), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [20ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [21ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [21ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [39ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [41ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [41ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [78ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [80ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [81ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [81ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [195ms] refreshing cache from master, [195ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [199ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [200ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [202ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [202ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [275ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [279ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [280ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [282ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [282ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [655ms] refreshing cache from master, [655ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [659ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [660ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [661ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [661ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [715ms] refreshing cache from master, [716ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [720ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@90623966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@60c01db9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856) -> (continuation of Deferred@1431237978 after retry RPC@1956158774) -> (continuation of Deferred@97962348 after retry RPC@890734029) -> (continuation of Deferred@775737757 after retry RPC@663486586) -> (continuation of Deferred@1336014299 after retry RPC@149529477), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b6ffffd -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856) -> (continuation of Deferred@1431237978 after retry RPC@1956158774) -> (continuation of Deferred@97962348 after retry RPC@890734029) -> (continuation of Deferred@775737757 after retry RPC@663486586) -> (continuation of Deferred@1336014299 after retry RPC@149529477))) after lookup
21:04:51.288 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:04:51.290 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:51.290 [DEBUG - kudu-nio-7] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:51.290 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:04:51.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.565909 11113 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:04:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.621546 11124 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 5 FOLLOWER]: Starting pre-election (detected failure of leader aa68dbc27a334b3bb725578a111601e4)
21:04:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.621963 11124 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:51.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.624212 11124 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:51.627274 10827 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111)
21:04:51.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:51.634084 10827 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711): Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111)
21:04:51.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.636693 11067 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "95d79f47ce794c54ab09ac6a3908316d" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" is_pre_election: true
21:04:51.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.637320 11067 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 95d79f47ce794c54ab09ac6a3908316d in term 4.
21:04:51.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.638447 10825 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, 95d79f47ce794c54ab09ac6a3908316d; no voters: aa68dbc27a334b3bb725578a111601e4
21:04:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.639142 11124 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 5 FOLLOWER]: Leader pre-election won for term 6
21:04:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.639441 11124 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 5 FOLLOWER]: Starting leader election (detected failure of leader aa68dbc27a334b3bb725578a111601e4)
21:04:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.639662 11124 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 5 FOLLOWER]: Advancing to term 6
21:04:51.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.643898 11124 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:51.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.645252 11124 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 6 election: Requested vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:04:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.646363 11067 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "95d79f47ce794c54ab09ac6a3908316d" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "18c80cf9fb014eb9bc13a9e176e05a00"
21:04:51.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.646898 11067 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 4 FOLLOWER]: Advancing to term 6
21:04:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:51.648970 10827 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711): Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111)
21:04:51.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.653379 11067 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 95d79f47ce794c54ab09ac6a3908316d in term 6.
21:04:51.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.654335 10825 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, 95d79f47ce794c54ab09ac6a3908316d; no voters: aa68dbc27a334b3bb725578a111601e4
21:04:51.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.655016 11124 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 6 FOLLOWER]: Leader election won for term 6
21:04:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.656461 11124 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 6 LEADER]: Becoming Leader. State: Replica: 95d79f47ce794c54ab09ac6a3908316d, State: Running, Role: LEADER
21:04:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.657231 11124 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:51.665449  9843 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d reported cstate change: term changed from 5 to 6, leader changed from aa68dbc27a334b3bb725578a111601e4 (127.9.107.1) to 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3). New cstate: current_term: 6 leader_uuid: "95d79f47ce794c54ab09ac6a3908316d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: HEALTHY } } }
21:04:52.266 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:52.266 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:52.267 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=16, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1699), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [20ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [21ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [21ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [39ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [41ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [41ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [78ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [80ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [81ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [81ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [195ms] refreshing cache from master, [195ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [199ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [200ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [202ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [202ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [275ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [279ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [280ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [282ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [282ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [655ms] refreshing cache from master, [655ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [659ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [660ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [661ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [661ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [715ms] refreshing cache from master, [716ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [720ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [721ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [723ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [723ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [1695ms] refreshing cache from master, [1695ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1698ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@90623966(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@60c01db9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856) -> (continuation of Deferred@1431237978 after retry RPC@1956158774) -> (continuation of Deferred@97962348 after retry RPC@890734029) -> (continuation of Deferred@775737757 after retry RPC@663486586) -> (continuation of Deferred@1336014299 after retry RPC@149529477) -> (continuation of Deferred@988554000 after retry RPC@2145147459), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b6ffffd -> wakeup thread Time-limited test -> (continuation of Deferred@465069674 after retry RPC@1794392018) -> (continuation of Deferred@572040446 after retry RPC@1767441856) -> (continuation of Deferred@1431237978 after retry RPC@1956158774) -> (continuation of Deferred@97962348 after retry RPC@890734029) -> (continuation of Deferred@775737757 after retry RPC@663486586) -> (continuation of Deferred@1336014299 after retry RPC@149529477) -> (continuation of Deferred@988554000 after retry RPC@2145147459))) after lookup
21:04:52.267 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:52.268 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] Successfully connected to peer
21:04:52.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:52.273641 11067 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 6 FOLLOWER]: Refusing update from remote peer 95d79f47ce794c54ab09ac6a3908316d: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
21:04:52.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:52.275693 11144 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [LEADER]: Connected to new peer: Peer: permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.001s
21:04:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:52.283166 10827 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d -> Peer aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711): Couldn't send request to peer aa68dbc27a334b3bb725578a111601e4. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:52.294 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xaadad0b6, L:/127.0.0.1:34906 - R:/127.9.107.3:42267] using TOKEN/null
21:04:52.385 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.107.1:34711
21:04:52.387 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:52.388670  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/logs
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.1:34711
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.1
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44535
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.1
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:52.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:52.698207 11157 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:52.698679 11157 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:52.699123 11157 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:52.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:52.732204 11157 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:52.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:52.733045 11157 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.1
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:52.772558 11157 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.1:34711
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.1
21:04:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44535
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.1
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/logs
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:52.773888 11157 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:52.775437 11157 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:52.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:52.787307 11163 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:53.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:52.788357 11164 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:53.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:53.941321 11166 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:53.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:53.943813 11165 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1151 milliseconds
21:04:53.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.943960 11157 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:04:53.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.945223 11157 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:53.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.947811 11157 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:53.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.949299 11157 hybrid_clock.cc:648] HybridClock initialized: now 1783544693949221 us; error 52 us; skew 500 ppm
21:04:53.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.950105 11157 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:53.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.956362 11157 webserver.cc:534] Webserver started at http://127.9.107.1:44535/ using document root <none> and password file <none>
21:04:53.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.957242 11157 fs_manager.cc:362] Metadata directory not provided
21:04:53.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.957473 11157 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:53.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.965660 11157 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.001s
21:04:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.970384 11178 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.971529 11157 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.002s
21:04:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.971881 11157 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa68dbc27a334b3bb725578a111601e4"
21:04:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:28 on dist-test-slave-qc6t"
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:53.973817 11157 fs_report.cc:389] FS layout report
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/data
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:54.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.021356 11157 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.022917 11157 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.023394 11157 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:54.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.026010 11157 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:54.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.031798 11185 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:04:54.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.039949 11157 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:54.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.040205 11157 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
21:04:54.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.040510 11157 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:54.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.045262 11157 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:54.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.045481 11157 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
21:04:54.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.045856 11185 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Bootstrap starting.
21:04:54.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.104025 11185 log.cc:826] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Log is configured to *not* fsync() on all Append() calls
21:04:54.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.183379 11185 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.184252 11185 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Bootstrap complete.
21:04:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.185886 11185 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Time spent bootstrapping tablet: real 0.140s	user 0.091s	sys 0.046s
21:04:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.203922 11185 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.206312 11185 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: aa68dbc27a334b3bb725578a111601e4, State: Initialized, Role: FOLLOWER
21:04:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.207538 11185 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.211540 11185 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Time spent starting tablet: real 0.025s	user 0.017s	sys 0.008s
21:04:54.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.218698 11157 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.1:34711
21:04:54.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.219002 11293 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.1:34711 every 8 connection(s)
21:04:54.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.221666 11157 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:04:54.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.222862  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 11157
21:04:54.224 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:54.268 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:54.270 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.107.3:42267
21:04:54.271 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:04:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.271358  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 10806
21:04:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.287619 11295 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:04:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.288141 11295 heartbeater.cc:461] Registering TS with master...
21:04:54.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.289491 11295 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:04:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.291833 11294 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:04:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.292209 11294 heartbeater.cc:461] Registering TS with master...
21:04:54.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.293097 11294 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:04:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.295933  9763 ts_manager.cc:194] Re-registered known tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:04:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.296954  9843 ts_manager.cc:194] Re-registered known tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:04:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.298384 11299 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:04:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.298722 11299 heartbeater.cc:461] Registering TS with master...
21:04:54.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.299597 11299 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:04:54.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.301149  9843 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.1:37997
21:04:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:54.302930  9688 ts_manager.cc:194] Re-registered known tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:04:54.311 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] handling channelInactive
21:04:54.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state READY due to: connection closed
21:04:54.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:54.312 [DEBUG - Time-limited test] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:54.312549 11302 negotiation.cc:336] Failed RPC negotiation. Trace:
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:04:54.259360 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.9.107.3:50389 (local address 127.9.107.1:34711)
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:04:54.283394 (+ 24034us) server_negotiation.cc:207] Beginning negotiation
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:04:54.283428 (+    34us) server_negotiation.cc:400] Waiting for connection header
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:04:54.283483 (+    55us) server_negotiation.cc:408] Connection header received
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:04:54.283682 (+   199us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:04:54.283716 (+    34us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:04:54.284096 (+   380us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:04:54.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:04:54.284292 (+   196us) server_negotiation.cc:227] Negotiated authn=SASL
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 21:04:54.310736 (+ 26444us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.9.107.3:50389: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":23350,"spinlock_wait_cycles":30592,"thread_start_us":22893,"threads_started":1}
21:04:54.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.314 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.327 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:54.327 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:54.327 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [14ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@827862337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@3a819103 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10578a9b -> wakeup thread Time-limited test)) after lookup
21:04:54.328 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:54.329 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.329 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.330 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.347 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:54.347 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:54.347 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [14ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [16ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [18ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [18ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@827862337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@3a819103 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10578a9b -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005))) after lookup
21:04:54.349 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:54.350 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.350 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.351 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.366 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:54.366 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:54.367 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=54), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [14ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [16ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [18ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [18ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [36ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [39ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [39ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [54ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@827862337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@3a819103 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005) -> (continuation of Deferred@826409368 after retry RPC@1642635818), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10578a9b -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005) -> (continuation of Deferred@826409368 after retry RPC@1642635818))) after lookup
21:04:54.367 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:54.368 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.368 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.369 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.406 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:54.406 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:54.406 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=94), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [14ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [16ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [18ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [18ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [36ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [39ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [39ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [54ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [55ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [57ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [57ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [93ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@827862337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@3a819103 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005) -> (continuation of Deferred@826409368 after retry RPC@1642635818) -> (continuation of Deferred@1527705841 after retry RPC@2136207432), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10578a9b -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005) -> (continuation of Deferred@826409368 after retry RPC@1642635818) -> (continuation of Deferred@1527705841 after retry RPC@2136207432))) after lookup
21:04:54.407 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:54.408 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.408 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.408 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.867 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:54.867 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:54.868 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=555), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [14ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [16ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [18ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [18ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [36ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [39ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [39ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [54ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [55ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [57ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [57ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [93ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [95ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [96ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [96ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [550ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [555ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@827862337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@3a819103 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005) -> (continuation of Deferred@826409368 after retry RPC@1642635818) -> (continuation of Deferred@1527705841 after retry RPC@2136207432) -> (continuation of Deferred@1935763937 after retry RPC@332404341), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10578a9b -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005) -> (continuation of Deferred@826409368 after retry RPC@1642635818) -> (continuation of Deferred@1527705841 after retry RPC@2136207432) -> (continuation of Deferred@1935763937 after retry RPC@332404341))) after lookup
21:04:54.868 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:54.869 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.869 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:54.869 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:55.226 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:55.227 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:55.227 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=915), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [14ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [16ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [18ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [18ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [36ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [39ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [39ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [54ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [55ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [57ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [57ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [93ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [95ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [96ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [96ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [550ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [555ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [556ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [557ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [557ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [910ms] refreshing cache from master, [910ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [914ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@827862337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@3a819103 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005) -> (continuation of Deferred@826409368 after retry RPC@1642635818) -> (continuation of Deferred@1527705841 after retry RPC@2136207432) -> (continuation of Deferred@1935763937 after retry RPC@332404341) -> (continuation of Deferred@658934726 after retry RPC@875971301), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10578a9b -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005) -> (continuation of Deferred@826409368 after retry RPC@1642635818) -> (continuation of Deferred@1527705841 after retry RPC@2136207432) -> (continuation of Deferred@1935763937 after retry RPC@332404341) -> (continuation of Deferred@658934726 after retry RPC@875971301))) after lookup
21:04:55.228 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:04:55.230 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:55.230 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:55.230 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:04:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.306202 11294 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:04:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.495362 11306 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 95d79f47ce794c54ab09ac6a3908316d)
21:04:55.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.495795 11306 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.498234 11306 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:55.501261 10999 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111)
21:04:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:55.505041 10999 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111)
21:04:55.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.510677 11248 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "aa68dbc27a334b3bb725578a111601e4" is_pre_election: true
21:04:55.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.511417 11248 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 18c80cf9fb014eb9bc13a9e176e05a00 in term 5.
21:04:55.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.512760 11000 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, aa68dbc27a334b3bb725578a111601e4; no voters: 95d79f47ce794c54ab09ac6a3908316d
21:04:55.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.513506 11306 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 6 FOLLOWER]: Leader pre-election won for term 7
21:04:55.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.513840 11306 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 95d79f47ce794c54ab09ac6a3908316d)
21:04:55.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.514146 11306 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 6 FOLLOWER]: Advancing to term 7
21:04:55.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.518599 11306 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:55.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.520313 11306 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 7 election: Requested vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.521083 11248 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "aa68dbc27a334b3bb725578a111601e4"
21:04:55.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.521700 11248 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 5 FOLLOWER]: Advancing to term 7
21:04:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:55.524324 10999 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111)
21:04:55.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.528376 11248 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 18c80cf9fb014eb9bc13a9e176e05a00 in term 7.
21:04:55.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.529367 11000 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, aa68dbc27a334b3bb725578a111601e4; no voters: 95d79f47ce794c54ab09ac6a3908316d
21:04:55.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.530218 11306 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 7 FOLLOWER]: Leader election won for term 7
21:04:55.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.531808 11306 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 7 LEADER]: Becoming Leader. State: Replica: 18c80cf9fb014eb9bc13a9e176e05a00, State: Running, Role: LEADER
21:04:55.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.532523 11306 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.541803  9843 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 reported cstate change: term changed from 6 to 7, leader changed from 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3) to 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2). New cstate: current_term: 7 leader_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: UNKNOWN } } }
21:04:55.827 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:55.827 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:55.827 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1515), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [2ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [2ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [14ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [16ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [18ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [18ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [36ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [39ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [39ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [54ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [55ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [57ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [57ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [93ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [95ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [96ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [96ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [550ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [555ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [556ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [557ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [557ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [910ms] refreshing cache from master, [910ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [914ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [916ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [918ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [918ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [1511ms] refreshing cache from master, [1511ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1514ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@827862337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@3a819103 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005) -> (continuation of Deferred@826409368 after retry RPC@1642635818) -> (continuation of Deferred@1527705841 after retry RPC@2136207432) -> (continuation of Deferred@1935763937 after retry RPC@332404341) -> (continuation of Deferred@658934726 after retry RPC@875971301) -> (continuation of Deferred@799100785 after retry RPC@972248922), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10578a9b -> wakeup thread Time-limited test -> (continuation of Deferred@2070915553 after retry RPC@1472264005) -> (continuation of Deferred@826409368 after retry RPC@1642635818) -> (continuation of Deferred@1527705841 after retry RPC@2136207432) -> (continuation of Deferred@1935763937 after retry RPC@332404341) -> (continuation of Deferred@658934726 after retry RPC@875971301) -> (continuation of Deferred@799100785 after retry RPC@972248922))) after lookup
21:04:55.828 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:55.830 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] Successfully connected to peer
21:04:55.847 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x90269bdb, L:/127.0.0.1:33772 - R:/127.9.107.2:42877] using TOKEN/null
21:04:55.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.857867 11248 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 7 FOLLOWER]: Refusing update from remote peer 18c80cf9fb014eb9bc13a9e176e05a00: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 26. (index mismatch)
21:04:55.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:55.858611 10999 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 -> Peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Couldn't send request to peer 95d79f47ce794c54ab09ac6a3908316d. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.859627 11311 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [LEADER]: Connected to new peer: Peer: permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
21:04:55.924 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.107.3:42267
21:04:55.924 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:04:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:55.925376  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:04:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data
21:04:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/logs
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.3:42267
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.3
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38725
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.3
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:56.247112 11334 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:56.247776 11334 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:56.248543 11334 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:56.299100 11334 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:56.299937 11334 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.3
21:04:56.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:56.353171 11334 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:56.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:56.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:04:56.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:56.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:56.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.3:42267
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.3
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38725
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.3
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/logs
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:04:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:04:56.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:56.354887 11334 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:56.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:56.357069 11334 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:56.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:56.372035 11340 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:56.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:56.612108 11290 debug-util.cc:398] Leaking SignalData structure 0x7b08000c4280 after lost signal to thread 11158
21:04:56.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:56.613363 11290 debug-util.cc:398] Leaking SignalData structure 0x7b08000bfce0 after lost signal to thread 11293
21:04:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:57.774796 11339 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 11334
21:04:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:58.130434 11339 kernel_stack_watchdog.cc:198] Thread 11334 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 399ms:
21:04:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:04:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:04:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:04:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:04:58.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:56.373427 11341 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:58.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:58.134606 11343 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:58.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:58.133739 11334 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.761s	user 0.622s	sys 1.011s
21:04:58.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:58.135454 11334 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.763s	user 0.622s	sys 1.011s
21:04:58.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:58.140403 11342 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1765 milliseconds
21:04:58.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.140583 11334 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:04:58.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.141819 11334 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.145820 11334 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:58.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.147225 11334 hybrid_clock.cc:648] HybridClock initialized: now 1783544698147117 us; error 119 us; skew 500 ppm
21:04:58.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.148219 11334 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.157687 11334 webserver.cc:534] Webserver started at http://127.9.107.3:38725/ using document root <none> and password file <none>
21:04:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.158624 11334 fs_manager.cc:362] Metadata directory not provided
21:04:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.158860 11334 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:58.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.166854 11334 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.000s	sys 0.004s
21:04:58.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.172772 11359 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:58.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.174033 11334 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.003s
21:04:58.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.174398 11334 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:58.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:04:58.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:33 on dist-test-slave-qc6t"
21:04:58.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.176343 11334 fs_report.cc:389] FS layout report
21:04:58.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/data
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:58.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.263379 11334 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.265141 11334 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.265795 11334 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:58.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:58.269937 10999 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 -> Peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Couldn't send request to peer 95d79f47ce794c54ab09ac6a3908316d. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:04:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.271011 11334 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.277387 11368 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:04:58.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.288931 11334 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:58.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.289196 11334 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.013s	user 0.001s	sys 0.001s
21:04:58.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.289529 11334 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.294257 11334 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.294497 11334 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
21:04:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.294872 11368 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Bootstrap starting.
21:04:58.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.369086 11368 log.cc:826] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Log is configured to *not* fsync() on all Append() calls
21:04:58.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.461565 11368 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:58.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.463033 11368 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Bootstrap complete.
21:04:58.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.464560 11368 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Time spent bootstrapping tablet: real 0.170s	user 0.120s	sys 0.039s
21:04:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.487985 11368 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.491719 11368 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95d79f47ce794c54ab09ac6a3908316d, State: Initialized, Role: FOLLOWER
21:04:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.493113 11368 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:04:58.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.508500 11368 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Time spent starting tablet: real 0.044s	user 0.025s	sys 0.019s
21:04:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.554772 11334 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.3:42267
21:04:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.555089 11476 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.3:42267 every 8 connection(s)
21:04:58.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.557524 11334 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:04:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.564134  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 11334
21:04:58.566 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:58.600 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:58.602 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.107.2:42877
21:04:58.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.603256  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 10977
21:04:58.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.610466 11479 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:04:58.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.610910 11479 heartbeater.cc:461] Registering TS with master...
21:04:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.612358 11479 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:04:58.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.618114  9689 ts_manager.cc:194] Re-registered known tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.625990 11477 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:04:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.626346 11477 heartbeater.cc:461] Registering TS with master...
21:04:58.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.627182 11477 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:04:58.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.627668 11478 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:04:58.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.628010 11478 heartbeater.cc:461] Registering TS with master...
21:04:58.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.628865 11478 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:04:58.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.630175  9844 ts_manager.cc:194] Re-registered known tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:58.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.632805  9844 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.3:53595
21:04:58.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:58.632905  9764 ts_manager.cc:194] Re-registered known tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:04:58.638 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] handling channelInactive
21:04:58.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state READY due to: connection closed
21:04:58.642 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:58.643 [DEBUG - Time-limited test] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:58.644 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.644 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.644 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.666 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:58.666 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:58.667 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@284634006(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2347dc4b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4dbdf735 -> wakeup thread Time-limited test)) after lookup
21:04:58.667 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:58.668 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.668 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.669 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.685 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:58.686 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:58.686 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [26ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@284634006(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2347dc4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4dbdf735 -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452))) after lookup
21:04:58.686 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:58.687 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.687 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.688 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.706 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:58.706 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:58.707 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=64), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [26ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [44ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [45ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [46ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [64ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@284634006(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2347dc4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452) -> (continuation of Deferred@1016232548 after retry RPC@1752251950), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4dbdf735 -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452) -> (continuation of Deferred@1016232548 after retry RPC@1752251950))) after lookup
21:04:58.707 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:58.708 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.708 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.708 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.746 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:58.746 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:58.747 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=104), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [26ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [44ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [45ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [46ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [64ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [65ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [66ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [66ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [100ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [104ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@284634006(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2347dc4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452) -> (continuation of Deferred@1016232548 after retry RPC@1752251950) -> (continuation of Deferred@338962757 after retry RPC@1968563619), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4dbdf735 -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452) -> (continuation of Deferred@1016232548 after retry RPC@1752251950) -> (continuation of Deferred@338962757 after retry RPC@1968563619))) after lookup
21:04:58.747 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:58.748 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.748 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.748 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.866 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:58.866 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:58.866 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=224), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [26ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [44ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [45ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [46ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [64ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [65ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [66ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [66ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [100ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [104ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [105ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [106ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [106ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [220ms] refreshing cache from master, [220ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [223ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@284634006(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2347dc4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452) -> (continuation of Deferred@1016232548 after retry RPC@1752251950) -> (continuation of Deferred@338962757 after retry RPC@1968563619) -> (continuation of Deferred@1922271532 after retry RPC@682689056), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4dbdf735 -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452) -> (continuation of Deferred@1016232548 after retry RPC@1752251950) -> (continuation of Deferred@338962757 after retry RPC@1968563619) -> (continuation of Deferred@1922271532 after retry RPC@682689056))) after lookup
21:04:58.867 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:58.868 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.868 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:58.869 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:59.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:59.636829 11477 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:04:59.787 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:04:59.787 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:04:59.788 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1145), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [26ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [44ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [45ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [46ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [64ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [65ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [66ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [66ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [100ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [104ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [105ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [106ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [106ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [220ms] refreshing cache from master, [220ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [223ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [225ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [227ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [227ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [1140ms] refreshing cache from master, [1141ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1144ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@284634006(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2347dc4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452) -> (continuation of Deferred@1016232548 after retry RPC@1752251950) -> (continuation of Deferred@338962757 after retry RPC@1968563619) -> (continuation of Deferred@1922271532 after retry RPC@682689056) -> (continuation of Deferred@1856313041 after retry RPC@1319479591), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4dbdf735 -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452) -> (continuation of Deferred@1016232548 after retry RPC@1752251950) -> (continuation of Deferred@338962757 after retry RPC@1968563619) -> (continuation of Deferred@1922271532 after retry RPC@682689056) -> (continuation of Deferred@1856313041 after retry RPC@1319479591))) after lookup
21:04:59.788 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:04:59.790 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:59.790 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:04:59.790 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.047655 11491 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 18c80cf9fb014eb9bc13a9e176e05a00)
21:05:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.048043 11491 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:00.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.050894 11491 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:05:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:00.053402 11179 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:05:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:00.056974 11179 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:05:00.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.062893 11431 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "aa68dbc27a334b3bb725578a111601e4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "95d79f47ce794c54ab09ac6a3908316d" is_pre_election: true
21:05:00.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.063517 11431 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aa68dbc27a334b3bb725578a111601e4 in term 6.
21:05:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.064778 11180 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 95d79f47ce794c54ab09ac6a3908316d, aa68dbc27a334b3bb725578a111601e4; no voters: 18c80cf9fb014eb9bc13a9e176e05a00
21:05:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.065451 11491 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 7 FOLLOWER]: Leader pre-election won for term 8
21:05:00.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.065711 11491 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 18c80cf9fb014eb9bc13a9e176e05a00)
21:05:00.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.066011 11491 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 7 FOLLOWER]: Advancing to term 8
21:05:00.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.070323 11491 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:00.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.071763 11491 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 8 election: Requested vote from peers 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:05:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.072824 11431 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "aa68dbc27a334b3bb725578a111601e4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:05:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.073499 11431 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 6 FOLLOWER]: Advancing to term 8
21:05:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:00.075124 11179 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:05:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.079420 11431 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aa68dbc27a334b3bb725578a111601e4 in term 8.
21:05:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.080308 11180 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 95d79f47ce794c54ab09ac6a3908316d, aa68dbc27a334b3bb725578a111601e4; no voters: 18c80cf9fb014eb9bc13a9e176e05a00
21:05:00.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.080915 11491 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 8 FOLLOWER]: Leader election won for term 8
21:05:00.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.082021 11491 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 8 LEADER]: Becoming Leader. State: Replica: aa68dbc27a334b3bb725578a111601e4, State: Running, Role: LEADER
21:05:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.082907 11491 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:00.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.091702  9844 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 reported cstate change: term changed from 7 to 8, leader changed from 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2) to aa68dbc27a334b3bb725578a111601e4 (127.9.107.1). New cstate: current_term: 8 leader_uuid: "aa68dbc27a334b3bb725578a111601e4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: UNKNOWN } } }
21:05:00.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.574221 11431 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 8 FOLLOWER]: Refusing update from remote peer aa68dbc27a334b3bb725578a111601e4: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
21:05:00.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:00.575023 11179 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 -> Peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Couldn't send request to peer 18c80cf9fb014eb9bc13a9e176e05a00. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:05:00.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.575730 11491 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
21:05:00.612 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:00.612 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:05:00.613 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1970), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [24ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [25ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [26ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [27ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [43ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [44ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [45ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [46ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [64ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [65ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [66ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [66ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [100ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [104ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [105ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [106ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [106ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [220ms] refreshing cache from master, [220ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [223ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [225ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [227ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [227ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [1140ms] refreshing cache from master, [1141ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1144ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [1146ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [1148ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [1148ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [1960ms] refreshing cache from master, [1961ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1969ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@284634006(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2347dc4b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452) -> (continuation of Deferred@1016232548 after retry RPC@1752251950) -> (continuation of Deferred@338962757 after retry RPC@1968563619) -> (continuation of Deferred@1922271532 after retry RPC@682689056) -> (continuation of Deferred@1856313041 after retry RPC@1319479591) -> (continuation of Deferred@1860766002 after retry RPC@224815449), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4dbdf735 -> wakeup thread Time-limited test -> (continuation of Deferred@1656013696 after retry RPC@1074955452) -> (continuation of Deferred@1016232548 after retry RPC@1752251950) -> (continuation of Deferred@338962757 after retry RPC@1968563619) -> (continuation of Deferred@1922271532 after retry RPC@682689056) -> (continuation of Deferred@1856313041 after retry RPC@1319479591) -> (continuation of Deferred@1860766002 after retry RPC@224815449))) after lookup
21:05:00.613 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:05:00.614 [DEBUG - kudu-nio-0] (Connection.java:757) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] Successfully connected to peer
21:05:00.635 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc914e225, L:/127.0.0.1:57642 - R:/127.9.107.1:34711] using TOKEN/null
21:05:00.677 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.107.2:42877
21:05:00.677 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:00.678900  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/logs
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.2:42877
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.2
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39083
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.2
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:05:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:05:00.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:05:00.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:05:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:01.003449 11519 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:05:01.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:01.003912 11519 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:05:01.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:01.004377 11519 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:05:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:01.037362 11519 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:05:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:01.038182 11519 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.2
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:01.075560 11519 tablet_server_runner.cc:78] Tablet server non-default flags:
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.2:42877
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.2
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39083
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.2
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/logs
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:05:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:05:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:01.076802 11519 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:05:01.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:01.078383 11519 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:05:01.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:01.090405 11525 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:02.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:01.091003 11526 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:01.094187 11528 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:02.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:02.283583 11527 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1190 milliseconds
21:05:02.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.283694 11519 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:05:02.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.284905 11519 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:05:02.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.287531 11519 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:05:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.288956 11519 hybrid_clock.cc:648] HybridClock initialized: now 1783544702288884 us; error 71 us; skew 500 ppm
21:05:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.289781 11519 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:05:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.295882 11519 webserver.cc:534] Webserver started at http://127.9.107.2:39083/ using document root <none> and password file <none>
21:05:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.296715 11519 fs_manager.cc:362] Metadata directory not provided
21:05:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.296895 11519 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:05:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.304664 11519 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
21:05:02.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.309510 11537 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:05:02.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.310592 11519 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.003s
21:05:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.310885 11519 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:05:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18c80cf9fb014eb9bc13a9e176e05a00"
21:05:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:30 on dist-test-slave-qc6t"
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.313586 11519 fs_report.cc:389] FS layout report
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/data
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:05:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:05:02.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.373641 11519 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:05:02.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.375106 11519 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:05:02.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.375516 11519 kserver.cc:163] Server-wide thread pool size limit: 3276
21:05:02.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.378013 11519 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:05:02.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.383453 11544 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:05:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.391263 11519 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:05:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.391505 11519 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.002s	sys 0.000s
21:05:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.391733 11519 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:05:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.396338 11519 ts_tablet_manager.cc:616] Registered 1 tablets
21:05:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.396512 11519 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
21:05:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.396976 11544 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap starting.
21:05:02.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.463788 11544 log.cc:826] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Log is configured to *not* fsync() on all Append() calls
21:05:02.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.556459 11519 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.2:42877
21:05:02.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.556615 11653 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.2:42877 every 8 connection(s)
21:05:02.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.559942 11519 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:05:02.568 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:05:02.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.567214  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 11519
21:05:02.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.570837 11544 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:05:02.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.571768 11544 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap complete.
21:05:02.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.573351 11544 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Time spent bootstrapping tablet: real 0.177s	user 0.138s	sys 0.031s
21:05:02.604 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:05:02.604 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.107.1:34711
21:05:02.605 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:05:02.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.605625  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 11157
21:05:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.629864 11655 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:05:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.630393 11655 heartbeater.cc:461] Registering TS with master...
21:05:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.631685 11655 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:05:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.631321 11544 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.634384 11544 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 18c80cf9fb014eb9bc13a9e176e05a00, State: Initialized, Role: FOLLOWER
21:05:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.635284 11544 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:02.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.641893 11656 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:05:02.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.642272 11656 heartbeater.cc:461] Registering TS with master...
21:05:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.643194 11656 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:05:02.646 [DEBUG - kudu-nio-0] (Connection.java:239) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] handling channelInactive
21:05:02.647 [DEBUG - kudu-nio-0] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state READY due to: connection closed
21:05:02.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:05:02.647 [DEBUG - Time-limited test] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:05:02.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.647883 11544 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Time spent starting tablet: real 0.074s	user 0.023s	sys 0.012s
21:05:02.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.649 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.650735  9764 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:05:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.651131 11654 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:05:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.651010  9688 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:05:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.651446 11654 heartbeater.cc:461] Registering TS with master...
21:05:02.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.652341 11654 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:05:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.655187  9844 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:05:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.657528  9844 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.2:36993
21:05:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:02.660115 11654 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:05:02.665 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:02.665 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:05:02.665 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [18ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1377477330(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@e9fd162 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45c70a33 -> wakeup thread Time-limited test)) after lookup
21:05:02.666 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:05:02.667 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.667 [DEBUG - kudu-nio-2] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.667 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.686 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:02.686 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:05:02.686 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [18ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [19ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [20ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [20ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [39ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1377477330(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@e9fd162 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1392519281 after retry RPC@787166896), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45c70a33 -> wakeup thread Time-limited test -> (continuation of Deferred@1392519281 after retry RPC@787166896))) after lookup
21:05:02.687 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:05:02.687 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.688 [DEBUG - kudu-nio-3] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.688 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.726 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:02.726 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:05:02.727 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [18ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [19ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [20ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [20ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [39ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [39ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [41ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [41ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [79ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1377477330(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@e9fd162 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1392519281 after retry RPC@787166896) -> (continuation of Deferred@578603526 after retry RPC@1440032335), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45c70a33 -> wakeup thread Time-limited test -> (continuation of Deferred@1392519281 after retry RPC@787166896) -> (continuation of Deferred@578603526 after retry RPC@1440032335))) after lookup
21:05:02.727 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:05:02.728 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.728 [DEBUG - kudu-nio-4] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.728 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.766 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:02.766 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:05:02.767 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=119), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [18ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [19ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [20ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [20ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [39ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [39ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [41ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [41ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [79ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [80ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [81ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [81ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [119ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1377477330(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@e9fd162 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1392519281 after retry RPC@787166896) -> (continuation of Deferred@578603526 after retry RPC@1440032335) -> (continuation of Deferred@2089608025 after retry RPC@1525209), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45c70a33 -> wakeup thread Time-limited test -> (continuation of Deferred@1392519281 after retry RPC@787166896) -> (continuation of Deferred@578603526 after retry RPC@1440032335) -> (continuation of Deferred@2089608025 after retry RPC@1525209))) after lookup
21:05:02.767 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:05:02.768 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.768 [DEBUG - kudu-nio-5] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.769 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.986 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:02.986 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:05:02.986 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=339), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [18ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [19ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [20ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [20ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [39ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [39ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [41ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [41ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [79ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [80ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [81ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [81ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [119ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [120ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [122ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [122ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [335ms] refreshing cache from master, [335ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [338ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1377477330(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@e9fd162 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1392519281 after retry RPC@787166896) -> (continuation of Deferred@578603526 after retry RPC@1440032335) -> (continuation of Deferred@2089608025 after retry RPC@1525209) -> (continuation of Deferred@1394394604 after retry RPC@1763666907), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45c70a33 -> wakeup thread Time-limited test -> (continuation of Deferred@1392519281 after retry RPC@787166896) -> (continuation of Deferred@578603526 after retry RPC@1440032335) -> (continuation of Deferred@2089608025 after retry RPC@1525209) -> (continuation of Deferred@1394394604 after retry RPC@1763666907))) after lookup
21:05:02.987 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:05:02.988 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.988 [DEBUG - kudu-nio-6] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state CONNECTING due to: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:02.988 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711
21:05:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.788019 11667 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 8 FOLLOWER]: Starting pre-election (detected failure of leader aa68dbc27a334b3bb725578a111601e4)
21:05:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.788450 11667 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:03.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.790763 11667 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:05:03.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:03.793385 11362 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111)
21:05:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:03.796552 11362 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711): Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111)
21:05:03.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.802450 11609 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "95d79f47ce794c54ab09ac6a3908316d" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" is_pre_election: true
21:05:03.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.803210 11609 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 95d79f47ce794c54ab09ac6a3908316d in term 7.
21:05:03.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.804562 11360 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, 95d79f47ce794c54ab09ac6a3908316d; no voters: aa68dbc27a334b3bb725578a111601e4
21:05:03.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.805253 11667 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 8 FOLLOWER]: Leader pre-election won for term 9
21:05:03.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.805549 11667 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 8 FOLLOWER]: Starting leader election (detected failure of leader aa68dbc27a334b3bb725578a111601e4)
21:05:03.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.805850 11667 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 8 FOLLOWER]: Advancing to term 9
21:05:03.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.810278 11667 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:03.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.811731 11667 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 9 election: Requested vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:05:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.812826 11609 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "95d79f47ce794c54ab09ac6a3908316d" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "18c80cf9fb014eb9bc13a9e176e05a00"
21:05:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.813380 11609 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 7 FOLLOWER]: Advancing to term 9
21:05:03.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:03.815121 11362 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711): Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111)
21:05:03.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.819711 11609 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 95d79f47ce794c54ab09ac6a3908316d in term 9.
21:05:03.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.820552 11360 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, 95d79f47ce794c54ab09ac6a3908316d; no voters: aa68dbc27a334b3bb725578a111601e4
21:05:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.821157 11667 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 9 FOLLOWER]: Leader election won for term 9
21:05:03.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.822664 11667 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 9 LEADER]: Becoming Leader. State: Replica: 95d79f47ce794c54ab09ac6a3908316d, State: Running, Role: LEADER
21:05:03.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.823503 11667 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:03.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:03.832432  9844 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d reported cstate change: term changed from 8 to 9, leader changed from aa68dbc27a334b3bb725578a111601e4 (127.9.107.1) to 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3). New cstate: current_term: 9 leader_uuid: "95d79f47ce794c54ab09ac6a3908316d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: HEALTHY } } }
21:05:04.206 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:04.206 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:04.207 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1559), Traces: [0ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [2ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [2ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [18ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [19ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [20ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [20ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [39ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [39ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [41ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [41ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [79ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [80ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [81ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [81ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [119ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [120ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [122ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [122ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [335ms] refreshing cache from master, [335ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [338ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [340ms] sending RPC to server aa68dbc27a334b3bb725578a111601e4, [341ms] received response from server aa68dbc27a334b3bb725578a111601e4: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [341ms] delaying RPC due to: Network error: Failed to connect to peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711): Connection refused: /127.9.107.1:34711, [1555ms] refreshing cache from master, [1555ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1559ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1377477330(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@e9fd162 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1392519281 after retry RPC@787166896) -> (continuation of Deferred@578603526 after retry RPC@1440032335) -> (continuation of Deferred@2089608025 after retry RPC@1525209) -> (continuation of Deferred@1394394604 after retry RPC@1763666907) -> (continuation of Deferred@27524823 after retry RPC@334503051), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45c70a33 -> wakeup thread Time-limited test -> (continuation of Deferred@1392519281 after retry RPC@787166896) -> (continuation of Deferred@578603526 after retry RPC@1440032335) -> (continuation of Deferred@2089608025 after retry RPC@1525209) -> (continuation of Deferred@1394394604 after retry RPC@1763666907) -> (continuation of Deferred@27524823 after retry RPC@334503051))) after lookup
21:05:04.207 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:05:04.208 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] Successfully connected to peer
21:05:04.228 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267] using TOKEN/null
21:05:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:04.237900 11609 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 9 FOLLOWER]: Refusing update from remote peer 95d79f47ce794c54ab09ac6a3908316d: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 34. (index mismatch)
21:05:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:04.239514 11672 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [LEADER]: Connected to new peer: Peer: permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
21:05:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:04.240435 11362 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d -> Peer aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711): Couldn't send request to peer aa68dbc27a334b3bb725578a111601e4. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.1:34711: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:05:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:04.256114 11688 mvcc.cc:204] Tried to move back new op lower bound from 7305399093817503744 to 7305399091548045312. Current Snapshot: MvccSnapshot[applied={T|T < 7305399093817503744}]
21:05:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:04.266654 11686 mvcc.cc:204] Tried to move back new op lower bound from 7305399108542214144 to 7305399106869788672. Current Snapshot: MvccSnapshot[applied={T|T < 7305399108542214144}]
21:05:04.312 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.107.1:34711
21:05:04.312 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:05:04.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:04.313493  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:05:04.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:05:04.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:05:04.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/logs
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.1:34711
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.1
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44535
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.1
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:05:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:05:04.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:04.641901 11695 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:05:04.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:04.642433 11695 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:05:04.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:04.643025 11695 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:05:04.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:04.675444 11695 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:05:04.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:04.676400 11695 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.1
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:04.712898 11695 tablet_server_runner.cc:78] Tablet server non-default flags:
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.1:34711
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.1
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44535
21:05:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.1
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/logs
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:05:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:04.714557 11695 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:05:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:04.716166 11695 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:05:04.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:04.727754 11701 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:05.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:04.729357 11702 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:05.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:04.731760 11704 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:05.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:05.860992 11703 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1130 milliseconds
21:05:05.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.861120 11695 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:05:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.862355 11695 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:05:05.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.864584 11695 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:05:05.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.866017 11695 hybrid_clock.cc:648] HybridClock initialized: now 1783544705865974 us; error 86 us; skew 500 ppm
21:05:05.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.866832 11695 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:05:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.873698 11695 webserver.cc:534] Webserver started at http://127.9.107.1:44535/ using document root <none> and password file <none>
21:05:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.874524 11695 fs_manager.cc:362] Metadata directory not provided
21:05:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.874737 11695 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:05:05.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.882666 11695 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.003s	sys 0.001s
21:05:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.887187 11713 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:05:05.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.888273 11695 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.004s
21:05:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.888562 11695 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:05:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa68dbc27a334b3bb725578a111601e4"
21:05:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:28 on dist-test-slave-qc6t"
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.890491 11695 fs_report.cc:389] FS layout report
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/wal
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/data
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:05:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:05:05.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.955937 11695 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:05:05.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.957394 11695 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:05:05.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.957813 11695 kserver.cc:163] Server-wide thread pool size limit: 3276
21:05:05.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.960412 11695 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:05:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.966172 11720 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:05:05.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.974524 11695 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:05:05.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.974798 11695 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.000s	sys 0.002s
21:05:05.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.975098 11695 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:05:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.979892 11695 ts_tablet_manager.cc:616] Registered 1 tablets
21:05:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.980086 11695 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.001s
21:05:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:05.980505 11720 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Bootstrap starting.
21:05:06.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.045184 11720 log.cc:826] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Log is configured to *not* fsync() on all Append() calls
21:05:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.143857 11695 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.1:34711
21:05:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.144032 11827 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.1:34711 every 8 connection(s)
21:05:06.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.147933 11695 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-0/data/info.pb
21:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.153910  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 11695
21:05:06.154 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:05:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.164083 11720 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:05:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.165057 11720 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Bootstrap complete.
21:05:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.166801 11720 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Time spent bootstrapping tablet: real 0.187s	user 0.134s	sys 0.047s
21:05:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.189400 11720 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.192456 11720 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: aa68dbc27a334b3bb725578a111601e4, State: Initialized, Role: FOLLOWER
21:05:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.194000 11720 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:06.202 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:05:06.205 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.107.3:42267
21:05:06.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:05:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.206337  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 11334
21:05:06.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:05:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.236311 11828 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:05:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.236764 11828 heartbeater.cc:461] Registering TS with master...
21:05:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.238082 11828 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:05:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.243077  9844 ts_manager.cc:194] Re-registered known tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:05:06.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.246714  9844 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.1:44429
21:05:06.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.265019 11828 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:05:06.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.266571 11720 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4: Time spent starting tablet: real 0.099s	user 0.069s	sys 0.022s
21:05:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.276825 11829 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:05:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.277189 11829 heartbeater.cc:461] Registering TS with master...
21:05:06.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.277846 11829 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:05:06.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.281064  9764 ts_manager.cc:194] Re-registered known tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:05:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.290984 11834 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:05:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.291339 11834 heartbeater.cc:461] Registering TS with master...
21:05:06.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.291924 11834 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:05:06.293 [ERROR - kudu-nio-7] (Connection.java:452) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267]
java.net.SocketException: Connection reset
	at java.base/sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394)
	at java.base/sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426)
	at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255)
	at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
	at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:156)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:840)
21:05:06.294 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state READY due to: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267]
21:05:06.295 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267]
21:05:06.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:06.295428  9688 ts_manager.cc:194] Re-registered known tserver with Master: aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711)
21:05:06.298 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] handling channelInactive
21:05:06.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:06.306 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:06.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=94), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [83ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [83ms] delaying RPC due to: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [94ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1044502013(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2d4682fa -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@174994c5 -> wakeup thread Time-limited test)) after lookup
21:05:06.307 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:05:06.308 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.308 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.308 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.326 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:06.326 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:06.327 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=114), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [83ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [83ms] delaying RPC due to: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [94ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [94ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [96ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [96ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [110ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [114ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1044502013(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2d4682fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@174994c5 -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455))) after lookup
21:05:06.327 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:05:06.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.328 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.328 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.366 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:06.366 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:06.366 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=154), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [83ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [83ms] delaying RPC due to: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [94ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [94ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [96ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [96ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [110ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [114ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [115ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [116ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [117ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [154ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1044502013(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2d4682fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455) -> (continuation of Deferred@97871515 after retry RPC@1335339907), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@174994c5 -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455) -> (continuation of Deferred@97871515 after retry RPC@1335339907))) after lookup
21:05:06.367 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:05:06.368 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.368 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.368 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.505 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:06.506 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:06.506 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=294), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [83ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [83ms] delaying RPC due to: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [94ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [94ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [96ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [96ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [110ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [114ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [115ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [116ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [117ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [154ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [155ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [156ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [156ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [290ms] refreshing cache from master, [290ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [293ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1044502013(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2d4682fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455) -> (continuation of Deferred@97871515 after retry RPC@1335339907) -> (continuation of Deferred@1024755753 after retry RPC@1328408507), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@174994c5 -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455) -> (continuation of Deferred@97871515 after retry RPC@1335339907) -> (continuation of Deferred@1024755753 after retry RPC@1328408507))) after lookup
21:05:06.506 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:05:06.507 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.507 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.507 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.546 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:06.546 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:06.546 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=334), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [83ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [83ms] delaying RPC due to: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [94ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [94ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [96ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [96ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [110ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [114ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [115ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [116ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [117ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [154ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [155ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [156ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [156ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [290ms] refreshing cache from master, [290ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [293ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [294ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [295ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [296ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [330ms] refreshing cache from master, [331ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [333ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1044502013(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2d4682fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455) -> (continuation of Deferred@97871515 after retry RPC@1335339907) -> (continuation of Deferred@1024755753 after retry RPC@1328408507) -> (continuation of Deferred@71411746 after retry RPC@611424046), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@174994c5 -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455) -> (continuation of Deferred@97871515 after retry RPC@1335339907) -> (continuation of Deferred@1024755753 after retry RPC@1328408507) -> (continuation of Deferred@71411746 after retry RPC@611424046))) after lookup
21:05:06.547 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:05:06.548 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.548 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:06.548 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:07.305 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:07.305 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)[L],aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:07.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1093), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [83ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [83ms] delaying RPC due to: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [94ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [94ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [96ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [96ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [110ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [114ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [115ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [116ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [117ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [154ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [155ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [156ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [156ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [290ms] refreshing cache from master, [290ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [293ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [294ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [295ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [296ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [330ms] refreshing cache from master, [331ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [333ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [335ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [336ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [336ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [1090ms] refreshing cache from master, [1090ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1093ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1044502013(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2d4682fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455) -> (continuation of Deferred@97871515 after retry RPC@1335339907) -> (continuation of Deferred@1024755753 after retry RPC@1328408507) -> (continuation of Deferred@71411746 after retry RPC@611424046) -> (continuation of Deferred@140301136 after retry RPC@549256594), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@174994c5 -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455) -> (continuation of Deferred@97871515 after retry RPC@1335339907) -> (continuation of Deferred@1024755753 after retry RPC@1328408507) -> (continuation of Deferred@71411746 after retry RPC@611424046) -> (continuation of Deferred@140301136 after retry RPC@549256594))) after lookup
21:05:07.306 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] connecting to peer
21:05:07.307 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:07.307 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] cleaning up while in state CONNECTING due to: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:07.307 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267
21:05:07.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.335018 11840 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 95d79f47ce794c54ab09ac6a3908316d)
21:05:07.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.335476 11840 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:07.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.338150 11840 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:05:07.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:07.343057 11539 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111)
21:05:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:07.347856 11539 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111)
21:05:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.350206 11783 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "aa68dbc27a334b3bb725578a111601e4" is_pre_election: true
21:05:07.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.351079 11783 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 18c80cf9fb014eb9bc13a9e176e05a00 in term 8.
21:05:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.352640 11540 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, aa68dbc27a334b3bb725578a111601e4; no voters: 95d79f47ce794c54ab09ac6a3908316d
21:05:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.353413 11840 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 9 FOLLOWER]: Leader pre-election won for term 10
21:05:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.353756 11840 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 95d79f47ce794c54ab09ac6a3908316d)
21:05:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.354104 11840 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 9 FOLLOWER]: Advancing to term 10
21:05:07.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.358791 11840 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:07.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.360421 11840 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 10 election: Requested vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:05:07.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.361156 11783 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "aa68dbc27a334b3bb725578a111601e4"
21:05:07.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.361646 11783 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 8 FOLLOWER]: Advancing to term 10
21:05:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:07.364173 11539 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111)
21:05:07.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.368479 11783 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 18c80cf9fb014eb9bc13a9e176e05a00 in term 10.
21:05:07.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.369596 11540 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18c80cf9fb014eb9bc13a9e176e05a00, aa68dbc27a334b3bb725578a111601e4; no voters: 95d79f47ce794c54ab09ac6a3908316d
21:05:07.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.370195 11840 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 10 FOLLOWER]: Leader election won for term 10
21:05:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.371675 11840 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 10 LEADER]: Becoming Leader. State: Replica: 18c80cf9fb014eb9bc13a9e176e05a00, State: Running, Role: LEADER
21:05:07.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.372707 11840 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.381603  9844 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 reported cstate change: term changed from 9 to 10, leader changed from 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3) to 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2). New cstate: current_term: 10 leader_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: UNKNOWN } } }
21:05:07.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.928412 11783 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 10 FOLLOWER]: Refusing update from remote peer 18c80cf9fb014eb9bc13a9e176e05a00: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
21:05:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:07.929809 11858 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [LEADER]: Connected to new peer: Peer: permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
21:05:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:07.954010 11539 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 -> Peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Couldn't send request to peer 95d79f47ce794c54ab09ac6a3908316d. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:05:10.425 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:10.426 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:10.426 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=4214), Traces: [0ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [83ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [83ms] delaying RPC due to: Network error: [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] unexpected exception from downstream on [id: 0x936ba633, L:/127.0.0.1:48048 - R:/127.9.107.3:42267], [90ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [94ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [94ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [96ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [96ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [110ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [114ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [115ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [116ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [117ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [154ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [155ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [156ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [156ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [290ms] refreshing cache from master, [290ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [293ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [294ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [295ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [296ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [330ms] refreshing cache from master, [331ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [333ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [335ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [336ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [336ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [1090ms] refreshing cache from master, [1090ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1093ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [1094ms] sending RPC to server 95d79f47ce794c54ab09ac6a3908316d, [1095ms] received response from server 95d79f47ce794c54ab09ac6a3908316d: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [1095ms] delaying RPC due to: Network error: Failed to connect to peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267): Connection refused: /127.9.107.3:42267, [4210ms] refreshing cache from master, [4210ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [4213ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@1044502013(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@2d4682fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455) -> (continuation of Deferred@97871515 after retry RPC@1335339907) -> (continuation of Deferred@1024755753 after retry RPC@1328408507) -> (continuation of Deferred@71411746 after retry RPC@611424046) -> (continuation of Deferred@140301136 after retry RPC@549256594) -> (continuation of Deferred@1791975563 after retry RPC@760762413), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@174994c5 -> wakeup thread Time-limited test -> (continuation of Deferred@1211130648 after retry RPC@327428455) -> (continuation of Deferred@97871515 after retry RPC@1335339907) -> (continuation of Deferred@1024755753 after retry RPC@1328408507) -> (continuation of Deferred@71411746 after retry RPC@611424046) -> (continuation of Deferred@140301136 after retry RPC@549256594) -> (continuation of Deferred@1791975563 after retry RPC@760762413))) after lookup
21:05:10.427 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:05:10.428 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] Successfully connected to peer
21:05:10.460 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0xca88938c, L:/127.0.0.1:46248 - R:/127.9.107.2:42877] using TOKEN/null
21:05:10.516 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.107.3:42267
21:05:10.516 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:05:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:10.518126  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:05:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:05:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:05:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data
21:05:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:05:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:05:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:05:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:05:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:05:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/logs
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.3:42267
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.3
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38725
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.3
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:05:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:05:10.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:10.522370 11539 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 -> Peer 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267): Couldn't send request to peer 95d79f47ce794c54ab09ac6a3908316d. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:05:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:10.817219 11888 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:05:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:10.817731 11888 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:05:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:10.818187 11888 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:05:10.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:10.851027 11888 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:05:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:10.852231 11888 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.3
21:05:10.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:10.888267 11888 tablet_server_runner.cc:78] Tablet server non-default flags:
21:05:10.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:05:10.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:05:10.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.3:42267
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.3
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38725
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.3
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/logs
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:05:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:10.889619 11888 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:05:10.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:10.891120 11888 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:05:10.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:10.903556 11895 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:10.903590 11894 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:12.218168 11896 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1313 milliseconds
21:05:12.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:12.219213 11898 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:12.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:12.222139 11888 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.318s	user 0.453s	sys 0.850s
21:05:12.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:12.222478 11888 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.319s	user 0.453s	sys 0.850s
21:05:12.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.222759 11888 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:05:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.227851 11888 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:05:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.235430 11888 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:05:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.237155 11888 hybrid_clock.cc:648] HybridClock initialized: now 1783544712237077 us; error 88 us; skew 500 ppm
21:05:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.238234 11888 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:05:12.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.246727 11888 webserver.cc:534] Webserver started at http://127.9.107.3:38725/ using document root <none> and password file <none>
21:05:12.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.247888 11888 fs_manager.cc:362] Metadata directory not provided
21:05:12.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.248185 11888 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:05:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.258330 11888 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.002s	sys 0.006s
21:05:12.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.264679 11908 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:05:12.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.266012 11888 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.003s
21:05:12.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.266366 11888 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:05:12.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:05:12.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:33 on dist-test-slave-qc6t"
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.268595 11888 fs_report.cc:389] FS layout report
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/wal
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/data
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:05:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:05:12.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.354733 11888 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:05:12.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.356304 11888 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:05:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.356786 11888 kserver.cc:163] Server-wide thread pool size limit: 3276
21:05:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.359387 11888 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:05:12.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.365001 11915 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:05:12.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.373205 11888 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:05:12.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.373492 11888 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
21:05:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.373808 11888 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:05:12.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.378423 11888 ts_tablet_manager.cc:616] Registered 1 tablets
21:05:12.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.378610 11888 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
21:05:12.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.378958 11915 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Bootstrap starting.
21:05:12.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.445924 11915 log.cc:826] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Log is configured to *not* fsync() on all Append() calls
21:05:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:12.495013 11539 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.3:42267: connect: Connection refused (error 111) [suppressed 10 similar messages]
21:05:12.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.548282 11888 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.3:42267
21:05:12.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.548395 12024 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.3:42267 every 8 connection(s)
21:05:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.550817 11888 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-2/data/info.pb
21:05:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.553622 11915 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:05:12.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:05:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.554682 11915 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Bootstrap complete.
21:05:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.556291 11915 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Time spent bootstrapping tablet: real 0.178s	user 0.110s	sys 0.062s
21:05:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.562850  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 11888
21:05:12.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.580519 11915 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:12.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.584491 11915 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95d79f47ce794c54ab09ac6a3908316d, State: Initialized, Role: FOLLOWER
21:05:12.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.585426 11915 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.600162 11915 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d: Time spent starting tablet: real 0.044s	user 0.029s	sys 0.012s
21:05:12.601 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:05:12.602 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.107.2:42877
21:05:12.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:05:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.603278  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 11519
21:05:12.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.614930 12027 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:05:12.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.615425 12027 heartbeater.cc:461] Registering TS with master...
21:05:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.616688 12027 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:05:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.616802 12025 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:05:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.617108 12025 heartbeater.cc:461] Registering TS with master...
21:05:12.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.617956 12025 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:05:12.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.619294 12026 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:05:12.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.619623 12026 heartbeater.cc:461] Registering TS with master...
21:05:12.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.620432 12026 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:05:12.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.620609  9844 ts_manager.cc:194] Re-registered known tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:05:12.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.623323  9844 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.3:42301
21:05:12.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.623695  9689 ts_manager.cc:194] Re-registered known tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:05:12.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.623684  9763 ts_manager.cc:194] Re-registered known tserver with Master: 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:05:12.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:12.627735 12025 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:05:12.636 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] handling channelInactive
21:05:12.636 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state READY due to: connection closed
21:05:12.636 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:05:12.637 [DEBUG - Time-limited test] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:05:12.638 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.638 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.638 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.646 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:12.646 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:12.646 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@296344564(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7e1f28d4 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@78b5ee1a -> wakeup thread Time-limited test)) after lookup
21:05:12.647 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:05:12.648 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.648 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.648 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.665 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:12.666 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:12.666 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [11ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [12ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [12ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [29ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@296344564(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7e1f28d4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@971699987 after retry RPC@1853360906), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@78b5ee1a -> wakeup thread Time-limited test -> (continuation of Deferred@971699987 after retry RPC@1853360906))) after lookup
21:05:12.666 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:05:12.667 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.668 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.668 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.706 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:12.706 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:12.707 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=70), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [11ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [12ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [12ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [29ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [30ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [32ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [32ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [70ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@296344564(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7e1f28d4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@971699987 after retry RPC@1853360906) -> (continuation of Deferred@1065397802 after retry RPC@471264104), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@78b5ee1a -> wakeup thread Time-limited test -> (continuation of Deferred@971699987 after retry RPC@1853360906) -> (continuation of Deferred@1065397802 after retry RPC@471264104))) after lookup
21:05:12.707 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:05:12.708 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.708 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.709 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.786 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:12.787 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:12.787 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=150), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [11ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [12ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [12ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [29ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [30ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [32ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [32ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [70ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [71ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [73ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [73ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [146ms] refreshing cache from master, [146ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [150ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@296344564(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7e1f28d4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@971699987 after retry RPC@1853360906) -> (continuation of Deferred@1065397802 after retry RPC@471264104) -> (continuation of Deferred@2076392357 after retry RPC@1353044510), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@78b5ee1a -> wakeup thread Time-limited test -> (continuation of Deferred@971699987 after retry RPC@1853360906) -> (continuation of Deferred@1065397802 after retry RPC@471264104) -> (continuation of Deferred@2076392357 after retry RPC@1353044510))) after lookup
21:05:12.788 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:05:12.788 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.789 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.789 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.829 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:12.829 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)[L],95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)]}]
21:05:12.829 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=192), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [11ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [12ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [12ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [29ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [30ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [32ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [32ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [70ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [71ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [73ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [73ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [146ms] refreshing cache from master, [146ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [150ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [151ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [153ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [153ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [186ms] refreshing cache from master, [186ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [192ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@296344564(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7e1f28d4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@971699987 after retry RPC@1853360906) -> (continuation of Deferred@1065397802 after retry RPC@471264104) -> (continuation of Deferred@2076392357 after retry RPC@1353044510) -> (continuation of Deferred@1880453771 after retry RPC@1560881733), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@78b5ee1a -> wakeup thread Time-limited test -> (continuation of Deferred@971699987 after retry RPC@1853360906) -> (continuation of Deferred@1065397802 after retry RPC@471264104) -> (continuation of Deferred@2076392357 after retry RPC@1353044510) -> (continuation of Deferred@1880453771 after retry RPC@1560881733))) after lookup
21:05:12.830 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] connecting to peer
21:05:12.832 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.833 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] cleaning up while in state CONNECTING due to: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:12.833 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877) for tablet a4634c0115174665ab84366629e41c73: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877
21:05:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.772660 12039 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:05:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.773159 12039 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.775715 12039 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers aa68dbc27a334b3bb725578a111601e4 (127.9.107.1:34711), 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:05:13.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:13.779181 11909 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:05:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:13.783044 11909 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:05:13.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.785667 11783 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "95d79f47ce794c54ab09ac6a3908316d" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "aa68dbc27a334b3bb725578a111601e4" is_pre_election: true
21:05:13.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.787031 11911 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 95d79f47ce794c54ab09ac6a3908316d; no voters: 18c80cf9fb014eb9bc13a9e176e05a00, aa68dbc27a334b3bb725578a111601e4
21:05:13.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.787704 12039 raft_consensus.cc:2749] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
21:05:13.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.847522 12044 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 18c80cf9fb014eb9bc13a9e176e05a00)
21:05:13.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.847963 12044 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:13.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.850438 12044 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:05:13.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:13.853694 11714 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:05:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:13.858115 11714 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:05:13.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.861804 11978 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "aa68dbc27a334b3bb725578a111601e4" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "95d79f47ce794c54ab09ac6a3908316d" is_pre_election: true
21:05:13.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.862435 11978 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aa68dbc27a334b3bb725578a111601e4 in term 9.
21:05:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.863662 11715 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 95d79f47ce794c54ab09ac6a3908316d, aa68dbc27a334b3bb725578a111601e4; no voters: 18c80cf9fb014eb9bc13a9e176e05a00
21:05:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.864431 12044 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 10 FOLLOWER]: Leader pre-election won for term 11
21:05:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.864732 12044 raft_consensus.cc:493] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 18c80cf9fb014eb9bc13a9e176e05a00)
21:05:13.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.864971 12044 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 10 FOLLOWER]: Advancing to term 11
21:05:13.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.869210 12044 raft_consensus.cc:515] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.870714 12044 leader_election.cc:290] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 11 election: Requested vote from peers 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877), 95d79f47ce794c54ab09ac6a3908316d (127.9.107.3:42267)
21:05:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.871798 11978 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a4634c0115174665ab84366629e41c73" candidate_uuid: "aa68dbc27a334b3bb725578a111601e4" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "95d79f47ce794c54ab09ac6a3908316d"
21:05:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.872283 11978 raft_consensus.cc:3060] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 9 FOLLOWER]: Advancing to term 11
21:05:13.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:13.873708 11714 leader_election.cc:336] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111)
21:05:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.878903 11978 raft_consensus.cc:2468] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aa68dbc27a334b3bb725578a111601e4 in term 11.
21:05:13.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.880040 11715 leader_election.cc:304] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 95d79f47ce794c54ab09ac6a3908316d, aa68dbc27a334b3bb725578a111601e4; no voters: 18c80cf9fb014eb9bc13a9e176e05a00
21:05:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.880865 12044 raft_consensus.cc:2804] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 11 FOLLOWER]: Leader election won for term 11
21:05:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.882359 12044 raft_consensus.cc:697] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [term 11 LEADER]: Becoming Leader. State: Replica: aa68dbc27a334b3bb725578a111601e4, State: Running, Role: LEADER
21:05:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.883165 12044 consensus_queue.cc:237] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:13.891630  9844 catalog_manager.cc:5696] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 reported cstate change: term changed from 10 to 11, leader changed from 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2) to aa68dbc27a334b3bb725578a111601e4 (127.9.107.1). New cstate: current_term: 11 leader_uuid: "aa68dbc27a334b3bb725578a111601e4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } health_report { overall_health: UNKNOWN } } }
21:05:14.228 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a4634c0115174665ab84366629e41c73 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909' with partition [<start>, <end>)
21:05:14.228 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a4634c0115174665ab84366629e41c73@[18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877),95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267),aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)[L]]}]
21:05:14.228 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a4634c0115174665ab84366629e41c73, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1591), Traces: [0ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [2ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [2ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [11ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [12ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [12ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [29ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [30ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [32ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [32ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [70ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [71ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [73ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [73ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [146ms] refreshing cache from master, [146ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [150ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [151ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [153ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [153ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [186ms] refreshing cache from master, [186ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [192ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, [194ms] sending RPC to server 18c80cf9fb014eb9bc13a9e176e05a00, [197ms] received response from server 18c80cf9fb014eb9bc13a9e176e05a00: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [197ms] delaying RPC due to: Network error: Failed to connect to peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877): Connection refused: /127.9.107.2:42877, [1586ms] refreshing cache from master, [1586ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.107.60:46817, [1591ms] Sub RPC GetTableLocations: received response from server master-127.9.107.60:46817: OK, deferred=Deferred@296344564(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f49f436b7a8@7e1f28d4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@971699987 after retry RPC@1853360906) -> (continuation of Deferred@1065397802 after retry RPC@471264104) -> (continuation of Deferred@2076392357 after retry RPC@1353044510) -> (continuation of Deferred@1880453771 after retry RPC@1560881733) -> (continuation of Deferred@78444615 after retry RPC@283357589), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@78b5ee1a -> wakeup thread Time-limited test -> (continuation of Deferred@971699987 after retry RPC@1853360906) -> (continuation of Deferred@1065397802 after retry RPC@471264104) -> (continuation of Deferred@2076392357 after retry RPC@1353044510) -> (continuation of Deferred@1880453771 after retry RPC@1560881733) -> (continuation of Deferred@78444615 after retry RPC@283357589))) after lookup
21:05:14.229 [DEBUG - kudu-nio-1] (Connection.java:748) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] connecting to peer
21:05:14.230 [DEBUG - kudu-nio-5] (Connection.java:757) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] Successfully connected to peer
21:05:14.279 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0xb5aa4879, L:/127.0.0.1:33324 - R:/127.9.107.1:34711] using TOKEN/null
21:05:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:14.288297 11714 consensus_peers.cc:597] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 -> Peer 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877): Couldn't send request to peer 18c80cf9fb014eb9bc13a9e176e05a00. Status: Network error: Client connection negotiation failed: client connection to 127.9.107.2:42877: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:05:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:14.289299 11978 raft_consensus.cc:1275] T a4634c0115174665ab84366629e41c73 P 95d79f47ce794c54ab09ac6a3908316d [term 11 FOLLOWER]: Refusing update from remote peer aa68dbc27a334b3bb725578a111601e4: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 42. (index mismatch)
21:05:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:14.290762 12044 consensus_queue.cc:1048] T a4634c0115174665ab84366629e41c73 P aa68dbc27a334b3bb725578a111601e4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
21:05:14.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:14.312127 12062 mvcc.cc:204] Tried to move back new op lower bound from 7305399149714833408 to 7305399148073144320. Current Snapshot: MvccSnapshot[applied={T|T < 7305399149714833408}]
21:05:14.357 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.107.2:42877
21:05:14.358 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:14.359328  9644 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu
21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data
21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:05:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/logs
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.2:42877
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.2
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39083
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.2
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:05:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:05:14.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:14.680497 12071 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:05:14.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:14.680977 12071 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:05:14.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:14.681483 12071 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:05:14.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:14.713443 12071 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:05:14.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:14.714287 12071 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.107.2
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:14.767984 12071 tablet_server_runner.cc:78] Tablet server non-default flags:
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.107.20:44843
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.107.2:42877
21:05:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.107.2
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39083
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.107.62:44527,127.9.107.61:39893,127.9.107.60:46817
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.107.2
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/logs
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:55 UTC on e7f111948823
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16878
21:05:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:05:14.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:14.769537 12071 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:05:14.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:14.771775 12071 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:05:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:14.787184 12078 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:15.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:15.772388 12021 debug-util.cc:398] Leaking SignalData structure 0x7b08000c9d80 after lost signal to thread 11889
21:05:15.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:15.773619 12021 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5c00 after lost signal to thread 12024
21:05:16.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:14.789426 12079 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:16.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:16.219380 12081 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:05:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:05:16.220331 12080 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1431 milliseconds
21:05:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.220481 12071 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:05:16.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.222172 12071 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:05:16.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.225433 12071 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:05:16.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.226969 12071 hybrid_clock.cc:648] HybridClock initialized: now 1783544716226830 us; error 138 us; skew 500 ppm
21:05:16.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.228072 12071 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:05:16.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.235937 12071 webserver.cc:534] Webserver started at http://127.9.107.2:39083/ using document root <none> and password file <none>
21:05:16.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.237062 12071 fs_manager.cc:362] Metadata directory not provided
21:05:16.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.237336 12071 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:05:16.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.248696 12071 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.006s	sys 0.000s
21:05:16.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.254184 12094 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:05:16.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.255259 12071 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
21:05:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.255532 12071 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data,/tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:05:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18c80cf9fb014eb9bc13a9e176e05a00"
21:05:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:30 on dist-test-slave-qc6t"
21:05:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.257393 12071 fs_report.cc:389] FS layout report
21:05:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:05:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:05:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/wal
21:05:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/data
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:05:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:05:16.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.323606 12071 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:05:16.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.325321 12071 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:05:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.325822 12071 kserver.cc:163] Server-wide thread pool size limit: 3276
21:05:16.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.329531 12071 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:05:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.341393 12101 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
21:05:16.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.357254 12071 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:05:16.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.357585 12071 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.018s	user 0.002s	sys 0.001s
21:05:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.357896 12071 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:05:16.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.364578 12071 ts_tablet_manager.cc:616] Registered 1 tablets
21:05:16.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.364846 12071 ts_tablet_manager.cc:595] Time spent register tablets: real 0.007s	user 0.004s	sys 0.000s
21:05:16.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.365302 12101 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap starting.
21:05:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.453583 12101 log.cc:826] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Log is configured to *not* fsync() on all Append() calls
21:05:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.595516 12101 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:05:16.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.596565 12101 tablet_bootstrap.cc:492] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Bootstrap complete.
21:05:16.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.598289 12101 ts_tablet_manager.cc:1403] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Time spent bootstrapping tablet: real 0.233s	user 0.160s	sys 0.066s
21:05:16.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.606514 12071 rpc_server.cc:307] RPC server started. Bound to: 127.9.107.2:42877
21:05:16.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.606772 12210 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.107.2:42877 every 8 connection(s)
21:05:16.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.610100 12071 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrAZmyJ/test-tmp/mini-kudu-cluster4149467949622657817/ts-1/data/info.pb
21:05:16.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.611331  9644 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu as pid 12071
21:05:16.612 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:05:16.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.625787 12101 raft_consensus.cc:359] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:16.632 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:05:16.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.649889 12101 raft_consensus.cc:740] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 18c80cf9fb014eb9bc13a9e176e05a00, State: Initialized, Role: FOLLOWER
21:05:16.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.652220 12101 consensus_queue.cc:260] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa68dbc27a334b3bb725578a111601e4" member_type: VOTER last_known_addr { host: "127.9.107.1" port: 34711 } } peers { permanent_uuid: "18c80cf9fb014eb9bc13a9e176e05a00" member_type: VOTER last_known_addr { host: "127.9.107.2" port: 42877 } } peers { permanent_uuid: "95d79f47ce794c54ab09ac6a3908316d" member_type: VOTER last_known_addr { host: "127.9.107.3" port: 42267 } }
21:05:16.662 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544673909, tablet=a4634c0115174665ab84366629e41c73, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:05:16.667 [DEBUG - main] (Connection.java:575) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] disconnecting while in state READY
21:05:16.668 [DEBUG - main] (Connection.java:575) [peer 95d79f47ce794c54ab09ac6a3908316d(127.9.107.3:42267)] disconnecting while in state TERMINATED
21:05:16.669 [DEBUG - main] (Connection.java:575) [peer 18c80cf9fb014eb9bc13a9e176e05a00(127.9.107.2:42877)] disconnecting while in state TERMINATED
21:05:16.669 [DEBUG - main] (Connection.java:575) [peer master-127.9.107.62:44527(127.9.107.62:44527)] disconnecting while in state READY
21:05:16.669 [DEBUG - main] (Connection.java:575) [peer master-127.9.107.60:46817(127.9.107.60:46817)] disconnecting while in state READY
21:05:16.669 [DEBUG - main] (Connection.java:575) [peer master-127.9.107.61:39893(127.9.107.61:39893)] disconnecting while in state READY
21:05:16.670 [DEBUG - kudu-nio-5] (Connection.java:239) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] handling channelInactive
21:05:16.670 [DEBUG - kudu-nio-5] (Connection.java:709) [peer aa68dbc27a334b3bb725578a111601e4(127.9.107.1:34711)] cleaning up while in state READY due to: connection closed
21:05:16.670 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.107.62:44527(127.9.107.62:44527)] handling channelInactive
21:05:16.670 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.9.107.60:46817(127.9.107.60:46817)] handling channelInactive
21:05:16.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.107.62:44527(127.9.107.62:44527)] cleaning up while in state READY due to: connection closed
21:05:16.671 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:05:16.674 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.107.60:46817(127.9.107.60:46817)] cleaning up while in state READY due to: connection closed
21:05:16.675 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.9.107.61:39893(127.9.107.61:39893)] handling channelInactive
21:05:16.675 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.9.107.61:39893(127.9.107.61:39893)] cleaning up while in state READY due to: connection closed
21:05:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.679121  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 11695
21:05:16.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.689530 12211 heartbeater.cc:344] Connected to a master server at 127.9.107.60:46817
21:05:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.690062 12211 heartbeater.cc:461] Registering TS with master...
21:05:16.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.703027 12211 heartbeater.cc:507] Master 127.9.107.60:46817 requested a full tablet report, sending...
21:05:16.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.707890  9843 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:05:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.711652  9843 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.107.2:57323
21:05:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.727629 12101 ts_tablet_manager.cc:1434] T a4634c0115174665ab84366629e41c73 P 18c80cf9fb014eb9bc13a9e176e05a00: Time spent starting tablet: real 0.129s	user 0.069s	sys 0.009s
21:05:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.727809 12216 heartbeater.cc:344] Connected to a master server at 127.9.107.62:44527
21:05:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.728178 12216 heartbeater.cc:461] Registering TS with master...
21:05:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.728696 12211 heartbeater.cc:499] Master 127.9.107.60:46817 was elected leader, sending a full tablet report...
21:05:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.728880 12216 heartbeater.cc:507] Master 127.9.107.62:44527 requested a full tablet report, sending...
21:05:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.732707  9689 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:05:16.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.746574 12215 heartbeater.cc:344] Connected to a master server at 127.9.107.61:39893
21:05:16.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.746898 12215 heartbeater.cc:461] Registering TS with master...
21:05:16.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.747433 12215 heartbeater.cc:507] Master 127.9.107.61:39893 requested a full tablet report, sending...
21:05:16.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.750176  9763 ts_manager.cc:194] Re-registered known tserver with Master: 18c80cf9fb014eb9bc13a9e176e05a00 (127.9.107.2:42877)
21:05:16.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.758013  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 12071
21:05:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.781996  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 11888
21:05:16.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.809805  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 9656
21:05:16.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.836633  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 9726
21:05:16.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:05:16.861061  9644 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskrAZmyJ/build/tsan/bin/kudu with pid 9801
21:05:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:05:17Z chronyd exiting

Time: 60.457

OK (1 test)

