JUnit version 4.13.2
.23:12:15.803 [INFO - main] (RandomUtils.java:49) Using random seed: 1783465935795
23:12:15.844 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
23:12:15.894 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
23:12:15.907 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasklgyMXj/test-tmp
23:12:15.909 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:12:15.913 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasklgyMXj/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:12:16.168 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

23:12:16.316 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:12:16.318 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

23:12:16.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:16.321273 20757 mini_oidc.cc:152] Starting JWKS server
23:12:16.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:16.325001 20757 webserver.cc:460] Webserver started at http://127.0.0.1:34703/ using document root <none> and password file <none>
23:12:16.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:16.325596 20757 mini_oidc.cc:189] Starting OIDC Discovery server
23:12:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:16.327420 20757 webserver.cc:460] Webserver started at http://127.0.0.1:33117/ using document root <none> and password file <none>
23:12:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:12:16Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
23:12:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:12:16Z Disabled control of system clock
23:12:16.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:16.379406 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:16.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:16.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/wal
23:12:16.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/data
23:12:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/logs
23:12:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/data/info.pb
23:12:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:16.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:16.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:16.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:16.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:16.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:16.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:16.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:16.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:12:16.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:16.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:12:16.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:12:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.126:41549
23:12:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.126
23:12:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:12:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:12:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34703/jwks.json with env {}
23:12:16.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.692713 20769 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:16.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.693337 20769 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:16.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.694123 20769 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:16.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.724628 20769 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:12:16.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.724931 20769 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:16.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.725203 20769 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:12:16.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.725477 20769 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:12:16.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.725739 20769 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:12:16.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.725973 20769 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34703/jwks.json
23:12:16.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:16.763216 20769 master_runner.cc:386] Master server non-default flags:
23:12:16.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:16.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:16.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:16.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:16.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/data
23:12:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/wal
23:12:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:12:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:12:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.126:41549
23:12:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:12:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:12:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34703/jwks.json
23:12:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/data/info.pb
23:12:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.126
23:12:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/logs
23:12:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:16.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:16.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:12:16.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:16.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:16.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:16.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:16.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:16.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:16.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:16.764489 20769 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:16.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:16.766348 20769 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.786268 20776 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:17.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.786219 20775 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:17.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:16.786262 20778 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:17.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:16.787874 20769 server_base.cc:1034] running on GCE node
23:12:17.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:17.975025 20769 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:17.978103 20769 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:17.979537 20769 hybrid_clock.cc:648] HybridClock initialized: now 1783465937979484 us; error 73 us; skew 500 ppm
23:12:17.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:17.980341 20769 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:17.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:17.992123 20769 webserver.cc:460] Webserver started at http://127.20.69.126:34477/ using document root <none> and password file <none>
23:12:17.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:17.993129 20769 fs_manager.cc:359] Metadata directory not provided
23:12:17.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:17.993323 20769 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:17.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:17.993810 20769 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:12:17.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:17.998731 20769 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/data/instance:
23:12:17.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aeb4fe52730c4670ba96bc1a07ab9127"
23:12:17.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:17 on dist-test-slave-l596"
23:12:18.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:17.999982 20769 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/wal/instance:
23:12:18.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aeb4fe52730c4670ba96bc1a07ab9127"
23:12:18.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:17 on dist-test-slave-l596"
23:12:18.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.007302 20769 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.001s	sys 0.008s
23:12:18.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.013245 20785 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:18.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.014698 20769 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
23:12:18.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.015028 20769 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/wal
23:12:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aeb4fe52730c4670ba96bc1a07ab9127"
23:12:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:17 on dist-test-slave-l596"
23:12:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.015515 20769 fs_report.cc:383] FS layout report
23:12:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/wal
23:12:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/wal
23:12:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/data/data
23:12:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:18.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:18.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.075937 20769 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.077569 20769 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:18.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.078037 20769 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:18.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.153080 20769 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.126:41549
23:12:18.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.153162 20836 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.126:41549 every 8 connection(s)
23:12:18.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.156049 20769 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/data/info.pb
23:12:18.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.161377 20837 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:12:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.166107 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 20769
23:12:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.166484 20757 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-0/wal/instance
23:12:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.169867 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/wal
23:12:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/data
23:12:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/logs
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/data/info.pb
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.125:36287
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.125
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:12:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:12:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34703/jwks.json with env {}
23:12:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.174368 20837 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } has no permanent_uuid. Determining permanent_uuid...
23:12:18.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.202699 20837 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } has no permanent_uuid. Determining permanent_uuid...
23:12:18.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.205224 20788 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.125:36287: connect: Connection refused (error 111)
23:12:18.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.209619 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.125:36287: Network error: Client connection negotiation failed: client connection to 127.20.69.125:36287: connect: Connection refused (error 111)
23:12:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.259130 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } attempt: 1
23:12:18.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.263652 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.125:36287: Network error: Client connection negotiation failed: client connection to 127.20.69.125:36287: connect: Connection refused (error 111)
23:12:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.332068 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } attempt: 2
23:12:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.336766 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.125:36287: Network error: Client connection negotiation failed: client connection to 127.20.69.125:36287: connect: Connection refused (error 111)
23:12:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.428225 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } attempt: 3
23:12:18.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.432686 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.125:36287: Network error: Client connection negotiation failed: client connection to 127.20.69.125:36287: connect: Connection refused (error 111)
23:12:18.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.491118 20839 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:18.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.491823 20839 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:18.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.492522 20839 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.523924 20839 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:12:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.524309 20839 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.524645 20839 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:12:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.524976 20839 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:12:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.525319 20839 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:12:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.525619 20839 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34703/jwks.json
23:12:18.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.561165 20839 master_runner.cc:386] Master server non-default flags:
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/data
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/wal
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.125:36287
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34703/jwks.json
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/data/info.pb
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.125
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/logs
23:12:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:12:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:18.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.562757 20839 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:18.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.565063 20839 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:18.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.576099 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } attempt: 4
23:12:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.581861 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.125:36287: Network error: Client connection negotiation failed: client connection to 127.20.69.125:36287: connect: Connection refused (error 111)
23:12:18.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.583746 20849 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.881433 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } attempt: 5
23:12:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.886150 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.125:36287: Network error: Client connection negotiation failed: client connection to 127.20.69.125:36287: connect: Connection refused (error 111)
23:12:19.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.433651 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } attempt: 6
23:12:19.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:19.441068 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.125:36287: Network error: Client connection negotiation failed: client connection to 127.20.69.125:36287: connect: Connection refused (error 111)
23:12:19.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.585125 20848 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:19.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:18.586483 20839 server_base.cc:1034] running on GCE node
23:12:19.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:18.586529 20851 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:19.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.806131 20839 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.809482 20839 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:19.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.810958 20839 hybrid_clock.cc:648] HybridClock initialized: now 1783465939810913 us; error 65 us; skew 500 ppm
23:12:19.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.811911 20839 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:19.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.818436 20839 webserver.cc:460] Webserver started at http://127.20.69.125:39299/ using document root <none> and password file <none>
23:12:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.819669 20839 fs_manager.cc:359] Metadata directory not provided
23:12:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.819900 20839 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.820439 20839 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:12:19.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.825431 20839 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/data/instance:
23:12:19.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9fd3cc16dbf477d88698543f4640b25"
23:12:19.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:19 on dist-test-slave-l596"
23:12:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.826802 20839 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/wal/instance:
23:12:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9fd3cc16dbf477d88698543f4640b25"
23:12:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:19 on dist-test-slave-l596"
23:12:19.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.834226 20839 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.001s
23:12:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.840229 20860 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:19.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.841322 20839 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
23:12:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.841717 20839 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/wal
23:12:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9fd3cc16dbf477d88698543f4640b25"
23:12:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:19 on dist-test-slave-l596"
23:12:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.842145 20839 fs_report.cc:383] FS layout report
23:12:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/wal
23:12:19.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/wal
23:12:19.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/data/data
23:12:19.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:19.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:19.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.894935 20839 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:19.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.896553 20839 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.897071 20839 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.969487 20839 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.125:36287
23:12:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.969710 20911 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.125:36287 every 8 connection(s)
23:12:19.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.972420 20839 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/data/info.pb
23:12:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.974936 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 20839
23:12:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.975451 20757 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-1/wal/instance
23:12:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.977792 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/wal
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/data
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/logs
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/data/info.pb
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.124:38915
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.124
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:12:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:12:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34703/jwks.json with env {}
23:12:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.978317 20912 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:12:19.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:19.990896 20912 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } has no permanent_uuid. Determining permanent_uuid...
23:12:20.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.016398 20912 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } has no permanent_uuid. Determining permanent_uuid...
23:12:20.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.033648 20912 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } has no permanent_uuid. Determining permanent_uuid...
23:12:20.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.036006 20863 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.040668 20912 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.090175 20912 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 1
23:12:20.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.094513 20912 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.162904 20912 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 2
23:12:20.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.167192 20912 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.258651 20912 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 3
23:12:20.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.263226 20912 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.297487 20914 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:20.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.298164 20914 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:20.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.298635 20914 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:20.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.329314 20914 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:12:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.329811 20914 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.330143 20914 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:12:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.330538 20914 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:12:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.330871 20914 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:12:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.331099 20914 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34703/jwks.json
23:12:20.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.366328 20914 master_runner.cc:386] Master server non-default flags:
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/data
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/wal
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.124:38915
23:12:20.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34703/jwks.json
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/data/info.pb
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.124
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/logs
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:20.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:20.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:12:20.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:20.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:20.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:20.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:20.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:20.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:20.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.367889 20914 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.370128 20914 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.391626 20925 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:20.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.406639 20912 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 4
23:12:20.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.410199 20912 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.501736 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } attempt: 7
23:12:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.517685 20837 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } has no permanent_uuid. Determining permanent_uuid...
23:12:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.524302 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.582765 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 1
23:12:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.587771 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.669222 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 2
23:12:20.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.673982 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.709865 20912 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 5
23:12:20.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.714175 20912 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.759415 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 3
23:12:20.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.764667 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.905130 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 4
23:12:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.910707 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:21.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.194193 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 5
23:12:21.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:21.199272 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.261752 20912 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 6
23:12:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:21.268985 20912 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:21.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.391665 20927 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:21.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:20.393407 20914 server_base.cc:1034] running on GCE node
23:12:21.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:20.392711 20924 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.657882 20914 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:21.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.660588 20914 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:21.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.661949 20914 hybrid_clock.cc:648] HybridClock initialized: now 1783465941661908 us; error 60 us; skew 500 ppm
23:12:21.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.662966 20914 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.669627 20914 webserver.cc:460] Webserver started at http://127.20.69.124:33795/ using document root <none> and password file <none>
23:12:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.670544 20914 fs_manager.cc:359] Metadata directory not provided
23:12:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.670727 20914 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.671104 20914 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:12:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.677371 20914 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/data/instance:
23:12:21.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1af80e215ab1412f8379dec863b32411"
23:12:21.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:21 on dist-test-slave-l596"
23:12:21.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.678510 20914 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/wal/instance:
23:12:21.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1af80e215ab1412f8379dec863b32411"
23:12:21.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:21 on dist-test-slave-l596"
23:12:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.685756 20914 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.001s
23:12:21.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.691998 20938 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:21.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.693264 20914 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.001s
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.693650 20914 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/wal
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1af80e215ab1412f8379dec863b32411"
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:21 on dist-test-slave-l596"
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.693996 20914 fs_report.cc:383] FS layout report
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/wal
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/wal
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/data/data
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:21.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.730530 20914 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:21.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.732499 20914 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.733158 20914 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:21.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.751718 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 6
23:12:21.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:21.759259 20837 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.69.124:38915: Network error: Client connection negotiation failed: client connection to 127.20.69.124:38915: connect: Connection refused (error 111)
23:12:21.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.814613 20914 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.124:38915
23:12:21.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.814713 20991 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.124:38915 every 8 connection(s)
23:12:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.817732 20914 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/data/info.pb
23:12:21.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.822342 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 20914
23:12:21.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.822799 20757 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/master-2/wal/instance
23:12:21.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.823710 20992 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:12:21.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.843704 20992 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } has no permanent_uuid. Determining permanent_uuid...
23:12:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.871834 20992 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } has no permanent_uuid. Determining permanent_uuid...
23:12:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.882917 20992 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } has no permanent_uuid. Determining permanent_uuid...
23:12:21.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.900049 20992 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411: Bootstrap starting.
23:12:21.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.907198 20992 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411: Neither blocks nor log segments found. Creating new log.
23:12:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.909636 20992 log.cc:824] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411: Log is configured to *not* fsync() on all Append() calls
23:12:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.914616 20992 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411: No bootstrap required, opened a new log
23:12:21.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.935640 20992 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } }
23:12:21.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.936259 20992 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:12:21.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.936595 20992 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1af80e215ab1412f8379dec863b32411, State: Initialized, Role: FOLLOWER
23:12:21.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.937407 20992 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } }
23:12:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.940356 21001 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } } }
23:12:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.941490 21001 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [sys.catalog]: This master's current role is: FOLLOWER
23:12:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.942106 20992 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [sys.catalog]: configured and running, proceeding with master startup.
23:12:21.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:21.967516 21012 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:12:21.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:21.967870 21012 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:12:21.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:21.989675 20944 tablet.cc:2369] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:12:21.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.963773 20914 master_runner.cc:186] Error getting master registration for 127.20.69.126:41549: OK, instance_id { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" instance_seqno: 1783465938132590 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:12:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:21.999781 20914 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:12:22.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.266985 21001 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:12:22.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.267581 21001 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } }
23:12:22.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.271245 21001 leader_election.cc:288] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aeb4fe52730c4670ba96bc1a07ab9127 (127.20.69.126:41549), c9fd3cc16dbf477d88698543f4640b25 (127.20.69.125:36287)
23:12:22.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.329653 20912 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 7
23:12:22.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.354897 20912 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25: Bootstrap starting.
23:12:22.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.363093 20912 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25: Neither blocks nor log segments found. Creating new log.
23:12:22.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.365831 20912 log.cc:824] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25: Log is configured to *not* fsync() on all Append() calls
23:12:22.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.372594 20912 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25: No bootstrap required, opened a new log
23:12:22.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.409262 20912 raft_consensus.cc:348] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } }
23:12:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.410321 20912 raft_consensus.cc:374] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:12:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.410666 20912 raft_consensus.cc:729] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9fd3cc16dbf477d88698543f4640b25, State: Initialized, Role: FOLLOWER
23:12:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.411743 20912 consensus_queue.cc:260] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } }
23:12:22.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.416200 21016 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } } }
23:12:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.417735 20912 sys_catalog.cc:564] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [sys.catalog]: configured and running, proceeding with master startup.
23:12:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.417846 21016 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [sys.catalog]: This master's current role is: FOLLOWER
23:12:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.418155 20887 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1af80e215ab1412f8379dec863b32411" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9fd3cc16dbf477d88698543f4640b25" is_pre_election: true
23:12:22.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.419071 20887 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1af80e215ab1412f8379dec863b32411 in term 0.
23:12:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.420696 20941 leader_election.cc:302] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1af80e215ab1412f8379dec863b32411, c9fd3cc16dbf477d88698543f4640b25; no voters: 
23:12:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.421977 21001 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:12:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.422389 21001 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:12:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.422791 21001 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 0 FOLLOWER]: Advancing to term 1
23:12:22.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.433161 21001 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } }
23:12:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.440281 20887 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1af80e215ab1412f8379dec863b32411" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9fd3cc16dbf477d88698543f4640b25"
23:12:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.441124 20887 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [term 0 FOLLOWER]: Advancing to term 1
23:12:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.449486 21001 leader_election.cc:288] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [CANDIDATE]: Term 1 election: Requested vote from peers aeb4fe52730c4670ba96bc1a07ab9127 (127.20.69.126:41549), c9fd3cc16dbf477d88698543f4640b25 (127.20.69.125:36287)
23:12:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.454092 20887 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1af80e215ab1412f8379dec863b32411 in term 1.
23:12:22.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.455194 20941 leader_election.cc:302] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1af80e215ab1412f8379dec863b32411, c9fd3cc16dbf477d88698543f4640b25; no voters: 
23:12:22.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.456113 21001 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 1 FOLLOWER]: Leader election won for term 1
23:12:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.458218 21001 raft_consensus.cc:686] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [term 1 LEADER]: Becoming Leader. State: Replica: 1af80e215ab1412f8379dec863b32411, State: Running, Role: LEADER
23:12:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.459157 21001 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } }
23:12:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.467310 21028 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
23:12:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.467419 21027 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1af80e215ab1412f8379dec863b32411. Latest consensus state: current_term: 1 leader_uuid: "1af80e215ab1412f8379dec863b32411" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } } }
23:12:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.468290 21027 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [sys.catalog]: This master's current role is: LEADER
23:12:22.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.482354 21030 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
23:12:22.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.491022 21030 catalog_manager.cc:1484] Initializing Kudu cluster ID...
23:12:22.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.512323 20887 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [term 1 FOLLOWER]: Refusing update from remote peer 1af80e215ab1412f8379dec863b32411: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:12:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.472477 20839 master_runner.cc:186] Error getting master registration for 127.20.69.126:41549: OK, instance_id { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" instance_seqno: 1783465938132590 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:12:22.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.514153 21001 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:12:22.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.514715 20839 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:12:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.535995 21016 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1af80e215ab1412f8379dec863b32411. Latest consensus state: current_term: 1 leader_uuid: "1af80e215ab1412f8379dec863b32411" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } } }
23:12:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.536782 21016 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [sys.catalog]: This master's current role is: FOLLOWER
23:12:22.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.542258 21027 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1af80e215ab1412f8379dec863b32411" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } } }
23:12:22.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.543107 21027 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [sys.catalog]: This master's current role is: LEADER
23:12:22.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.545830 21030 catalog_manager.cc:1347] Generated new cluster ID: 9a4e1db3073b48169abe887844dec28e
23:12:22.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.546144 21030 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
23:12:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.548261 21016 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1af80e215ab1412f8379dec863b32411" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } } }
23:12:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.549003 21016 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25 [sys.catalog]: This master's current role is: FOLLOWER
23:12:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.577881 21030 catalog_manager.cc:1370] Generated new certificate authority record
23:12:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.580506 21030 catalog_manager.cc:1504] Loading token signing keys...
23:12:22.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.608485 21030 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411: Generated new TSK 0
23:12:22.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.609756 21030 catalog_manager.cc:1514] Initializing in-progress tserver states...
23:12:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.641288 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/logs
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.65:0
23:12:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.65
23:12:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.65
23:12:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:12:22.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.792685 20837 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } attempt: 7
23:12:22.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.819787 20837 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127: Bootstrap starting.
23:12:22.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.827894 20837 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127: Neither blocks nor log segments found. Creating new log.
23:12:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.831141 20837 log.cc:824] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127: Log is configured to *not* fsync() on all Append() calls
23:12:22.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.838093 20837 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127: No bootstrap required, opened a new log
23:12:22.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.875556 20837 raft_consensus.cc:348] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } }
23:12:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.876971 20837 raft_consensus.cc:374] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:12:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.877570 20837 raft_consensus.cc:729] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aeb4fe52730c4670ba96bc1a07ab9127, State: Initialized, Role: FOLLOWER
23:12:22.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.878870 20837 consensus_queue.cc:260] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } }
23:12:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.894546 20811 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1af80e215ab1412f8379dec863b32411" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aeb4fe52730c4670ba96bc1a07ab9127"
23:12:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.894953 20810 raft_consensus.cc:3037] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [term 0 FOLLOWER]: Advancing to term 1
23:12:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.896481 20837 sys_catalog.cc:564] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [sys.catalog]: configured and running, proceeding with master startup.
23:12:22.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.901749 20812 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1af80e215ab1412f8379dec863b32411" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" is_pre_election: true
23:12:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.904399 20810 raft_consensus.cc:1261] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [term 1 FOLLOWER]: Refusing update from remote peer 1af80e215ab1412f8379dec863b32411: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:12:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.892836 21039 sys_catalog.cc:455] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } } }
23:12:22.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.910681 21027 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [LEADER]: Connected to new peer: Peer: permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:12:22.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:22.916043 21039 sys_catalog.cc:458] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [sys.catalog]: This master's current role is: FOLLOWER
23:12:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.027215 21041 sys_catalog.cc:455] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1af80e215ab1412f8379dec863b32411" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } } }
23:12:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.028066 21041 sys_catalog.cc:458] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [sys.catalog]: This master's current role is: FOLLOWER
23:12:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.028659 21039 sys_catalog.cc:455] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1af80e215ab1412f8379dec863b32411. Latest consensus state: current_term: 1 leader_uuid: "1af80e215ab1412f8379dec863b32411" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } } }
23:12:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.029615 21039 sys_catalog.cc:458] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127 [sys.catalog]: This master's current role is: FOLLOWER
23:12:23.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.066910 21055 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
23:12:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:23.119673 21036 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:23.120242 21036 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:23.120883 21036 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:23.152753 21036 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:23.153945 21036 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.65
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.189941 21036 tablet_server_runner.cc:78] Tablet server non-default flags:
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.65:0
23:12:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.65
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:23.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.65
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/logs
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.191505 21036 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.193635 21036 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:23.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:23.231559 21062 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.489997 21028 catalog_manager.cc:1259] Loaded cluster ID: 9a4e1db3073b48169abe887844dec28e
23:12:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.494897 21028 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25: loading cluster ID for follower catalog manager: success
23:12:23.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.503165 21028 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25: acquiring CA information for follower catalog manager: success
23:12:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.520545 21028 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c9fd3cc16dbf477d88698543f4640b25: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:12:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.531016 21027 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1af80e215ab1412f8379dec863b32411" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aeb4fe52730c4670ba96bc1a07ab9127" member_type: VOTER last_known_addr { host: "127.20.69.126" port: 41549 } } peers { permanent_uuid: "c9fd3cc16dbf477d88698543f4640b25" member_type: VOTER last_known_addr { host: "127.20.69.125" port: 36287 } } peers { permanent_uuid: "1af80e215ab1412f8379dec863b32411" member_type: VOTER last_known_addr { host: "127.20.69.124" port: 38915 } } }
23:12:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:23.532497 21027 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1af80e215ab1412f8379dec863b32411 [sys.catalog]: This master's current role is: LEADER
23:12:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:24.094579 21055 catalog_manager.cc:1259] Loaded cluster ID: 9a4e1db3073b48169abe887844dec28e
23:12:24.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:24.095209 21055 catalog_manager.cc:1552] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127: loading cluster ID for follower catalog manager: success
23:12:24.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:24.103659 21055 catalog_manager.cc:1574] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127: acquiring CA information for follower catalog manager: success
23:12:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:24.109865 21055 catalog_manager.cc:1602] T 00000000000000000000000000000000 P aeb4fe52730c4670ba96bc1a07ab9127: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:12:25.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:23.233062 21061 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:25.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:25.376446 21063 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2145 milliseconds
23:12:25.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.377223 21036 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:12:25.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:23.231630 21064 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:25.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.383293 21036 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.386336 21036 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:25.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.388074 21036 hybrid_clock.cc:648] HybridClock initialized: now 1783465945388019 us; error 69 us; skew 500 ppm
23:12:25.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.389169 21036 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:25.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.406824 21036 webserver.cc:460] Webserver started at http://127.20.69.65:45683/ using document root <none> and password file <none>
23:12:25.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.408140 21036 fs_manager.cc:359] Metadata directory not provided
23:12:25.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.408417 21036 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.409013 21036 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:12:25.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.416186 21036 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/instance:
23:12:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d65993ec41674e8f976e3ecbb62149e4"
23:12:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:25 on dist-test-slave-l596"
23:12:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.417764 21036 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal/instance:
23:12:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d65993ec41674e8f976e3ecbb62149e4"
23:12:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:25 on dist-test-slave-l596"
23:12:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.428511 21036 fs_manager.cc:691] Time spent creating directory manager: real 0.010s	user 0.009s	sys 0.000s
23:12:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.436508 21076 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.438019 21036 fs_manager.cc:725] Time spent opening block manager: real 0.005s	user 0.007s	sys 0.000s
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.438431 21036 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d65993ec41674e8f976e3ecbb62149e4"
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:25 on dist-test-slave-l596"
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.438966 21036 fs_report.cc:383] FS layout report
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/data
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.487900 21036 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.489969 21036 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.490561 21036 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:25.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.494096 21036 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:12:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.500108 21036 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:12:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.500401 21036 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:12:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.500697 21036 ts_tablet_manager.cc:610] Registered 0 tablets
23:12:25.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.500909 21036 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:12:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.746284 21036 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.65:45117
23:12:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.746404 21189 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.65:45117 every 8 connection(s)
23:12:25.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.750489 21036 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:12:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.751109 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 21036
23:12:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.751580 20757 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal/instance
23:12:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.772934 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data
23:12:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/logs
23:12:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:12:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.66:0
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.66
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.66
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:12:25.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.848234 21191 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:12:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.848749 21191 heartbeater.cc:464] Registering TS with master...
23:12:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.849901 21191 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:12:25.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.854346 20877 ts_manager.cc:194] Registered new tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:25.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.861742 21190 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:12:25.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.862107 21190 heartbeater.cc:464] Registering TS with master...
23:12:25.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.862985 21190 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:12:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.865051 21192 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:12:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.865459 21192 heartbeater.cc:464] Registering TS with master...
23:12:25.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.866107 20957 ts_manager.cc:194] Registered new tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:25.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.866482 21192 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:12:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.869710 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.65:50743
23:12:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:25.869810 20802 ts_manager.cc:194] Registered new tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:26.240736 21198 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:26.241328 21198 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:26.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:26.242012 21198 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:26.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:26.289126 21198 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:26.290192 21198 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.66
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:26.326015 21198 tablet_server_runner.cc:78] Tablet server non-default flags:
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.66:0
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.66
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.66
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/logs
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:26.327553 21198 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:26.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:26.329619 21198 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:26.358299 21207 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:26.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:26.877457 21190 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:12:28.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:26.359282 21208 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:28.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:27.749392 21206 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 21198
23:12:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.217916 21206 kernel_stack_watchdog.cc:198] Thread 21198 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 389ms:
23:12:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:12:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:12:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:12:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:12:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.218233 21198 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.861s	user 0.654s	sys 1.084s
23:12:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.218547 21198 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.861s	user 0.654s	sys 1.084s
23:12:28.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.220223 21210 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.221942 21209 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1860 milliseconds
23:12:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.222007 21198 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:12:28.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.223258 21198 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:28.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.225557 21198 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:28.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.227015 21198 hybrid_clock.cc:648] HybridClock initialized: now 1783465948226968 us; error 46 us; skew 500 ppm
23:12:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.227980 21198 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:28.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.235064 21198 webserver.cc:460] Webserver started at http://127.20.69.66:37625/ using document root <none> and password file <none>
23:12:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.236142 21198 fs_manager.cc:359] Metadata directory not provided
23:12:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.236418 21198 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.236963 21198 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:12:28.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.242354 21198 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/instance:
23:12:28.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4da2752128f4adba064520c15942e13"
23:12:28.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:28 on dist-test-slave-l596"
23:12:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.243691 21198 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal/instance:
23:12:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4da2752128f4adba064520c15942e13"
23:12:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:28 on dist-test-slave-l596"
23:12:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.251703 21198 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.004s	sys 0.004s
23:12:28.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.257350 21218 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.258745 21198 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
23:12:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.259044 21198 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4da2752128f4adba064520c15942e13"
23:12:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:28 on dist-test-slave-l596"
23:12:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.259348 21198 fs_report.cc:383] FS layout report
23:12:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/data
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:28.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:28.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.301707 21198 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:28.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.303162 21198 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:28.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.303579 21198 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.306253 21198 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:12:28.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.310459 21198 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:12:28.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.310750 21198 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:12:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.311002 21198 ts_tablet_manager.cc:610] Registered 0 tablets
23:12:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.311137 21198 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:12:28.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.467398 21198 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.66:46673
23:12:28.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.467594 21331 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.66:46673 every 8 connection(s)
23:12:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.470119 21198 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:12:28.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.474692 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 21198
23:12:28.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.475363 20757 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal/instance
23:12:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.497093 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/logs
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.67:0
23:12:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.67
23:12:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.67
23:12:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:12:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.522709 21336 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:12:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.523262 21336 heartbeater.cc:464] Registering TS with master...
23:12:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.523717 21332 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:12:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.523994 21332 heartbeater.cc:464] Registering TS with master...
23:12:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.524670 21336 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:12:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.524801 21332 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:12:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.527840 20957 ts_manager.cc:194] Registered new tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.528944 20802 ts_manager.cc:194] Registered new tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:28.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.530176 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.66:58743
23:12:28.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.536885 21333 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:12:28.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.537266 21333 heartbeater.cc:464] Registering TS with master...
23:12:28.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.537974 21333 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:12:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.540520 20877 ts_manager.cc:194] Registered new tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:28.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.847533 21342 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:28.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.848184 21342 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.848906 21342 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:28.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.882504 21342 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.883306 21342 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.67
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.918761 21342 tablet_server_runner.cc:78] Tablet server non-default flags:
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.67:0
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.67
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.67
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/logs
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.920073 21342 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.921854 21342 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:28.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.942551 21348 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:29.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:29.533802 21332 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:12:30.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.942709 21349 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:28.943151 21351 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:28.945479 21342 server_base.cc:1034] running on GCE node
23:12:30.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.154111 21342 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:30.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.156872 21342 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.158756 21342 hybrid_clock.cc:648] HybridClock initialized: now 1783465950158712 us; error 46 us; skew 500 ppm
23:12:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.159814 21342 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:30.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.167316 21342 webserver.cc:460] Webserver started at http://127.20.69.67:39541/ using document root <none> and password file <none>
23:12:30.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.168368 21342 fs_manager.cc:359] Metadata directory not provided
23:12:30.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.168625 21342 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:30.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.169162 21342 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:12:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.174322 21342 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/instance:
23:12:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c75eda9306bf472caf0f597e71854313"
23:12:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:30 on dist-test-slave-l596"
23:12:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.175500 21342 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal/instance:
23:12:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c75eda9306bf472caf0f597e71854313"
23:12:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:30 on dist-test-slave-l596"
23:12:30.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.183442 21342 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.004s	sys 0.004s
23:12:30.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.189242 21358 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:30.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.190454 21342 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
23:12:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.190825 21342 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c75eda9306bf472caf0f597e71854313"
23:12:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:30 on dist-test-slave-l596"
23:12:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.191222 21342 fs_report.cc:383] FS layout report
23:12:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/data
23:12:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.239331 21342 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:30.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.240895 21342 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:30.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.241439 21342 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:30.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.244390 21342 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:12:30.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.249468 21342 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:12:30.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.249776 21342 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.001s	sys 0.000s
23:12:30.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.250123 21342 ts_tablet_manager.cc:610] Registered 0 tablets
23:12:30.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.250322 21342 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:12:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.397315 21342 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.67:43603
23:12:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.397449 21471 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.67:43603 every 8 connection(s)
23:12:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.400019 21342 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:12:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.405791 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 21342
23:12:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.406356 20757 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal/instance
23:12:30.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.464756 21472 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:12:30.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.464893 21473 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:12:30.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.465291 21472 heartbeater.cc:464] Registering TS with master...
23:12:30.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.465386 21473 heartbeater.cc:464] Registering TS with master...
23:12:30.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.466738 21472 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:12:30.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.466815 21474 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:12:30.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.467010 21473 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:12:30.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.467198 21474 heartbeater.cc:464] Registering TS with master...
23:12:30.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.467986 21474 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:12:30.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.469954 20957 ts_manager.cc:194] Registered new tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:12:30.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.470724 20877 ts_manager.cc:194] Registered new tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:12:30.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.470870 20802 ts_manager.cc:194] Registered new tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:12:30.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.472389 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.67:55945
23:12:30.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:30.484790 20757 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
23:12:30.495 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:12:30.496 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

23:12:30.506 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.20.69.126"
      port: 41549
    }
    bound_http_address {
      host: "127.20.69.126"
      port: 34477
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.20.69.125"
      port: 36287
    }
    bound_http_address {
      host: "127.20.69.125"
      port: 39299
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.20.69.124"
      port: 38915
    }
    bound_http_address {
      host: "127.20.69.124"
      port: 33795
    }
  }
}

23:12:30.512 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

23:12:30.514 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.20.69.65"
      port: 45117
    }
    bound_http_address {
      host: "127.20.69.65"
      port: 45683
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.20.69.66"
      port: 46673
    }
    bound_http_address {
      host: "127.20.69.66"
      port: 37625
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.20.69.67"
      port: 43603
    }
    bound_http_address {
      host: "127.20.69.67"
      port: 39541
    }
  }
}

23:12:30.515 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
23:12:30.696 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
23:12:30.906 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.69.125:36287(127.20.69.125:36287)] connecting to peer
23:12:30.965 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.69.126:41549(127.20.69.126:41549)] connecting to peer
23:12:30.973 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.69.124:38915(127.20.69.124:38915)] connecting to peer
23:12:31.004 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.69.126:41549(127.20.69.126:41549)] Successfully connected to peer
23:12:31.015 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.69.125:36287(127.20.69.125:36287)] Successfully connected to peer
23:12:31.087 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.69.124:38915(127.20.69.124:38915)] Successfully connected to peer
23:12:31.113 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.69.125
23:12:31.113 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.69.126
23:12:31.243 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.69.124
23:12:31.368 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x597bcd03, L:/127.0.0.1:42488 - R:/127.20.69.124:38915] using SASL/PLAIN
23:12:31.368 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x237a2e41, L:/127.0.0.1:33860 - R:/127.20.69.126:41549] using SASL/PLAIN
23:12:31.373 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xced16125, L:/127.0.0.1:41924 - R:/127.20.69.125:36287] using SASL/PLAIN
23:12:31.413 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
23:12:31.414 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=562), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.69.125:36287, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.69.126:41549, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.69.124:38915, [527ms] Sub RPC ConnectToMaster: received response from server master-127.20.69.126:41549: OK, [528ms] Sub RPC ConnectToMaster: received response from server master-127.20.69.124:38915: OK, deferred=null) after lookup
23:12:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.477108 21472 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:12:31.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.479948 20957 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42488:
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833"
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:12:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:12:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:12:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:12:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:12:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
23:12:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
23:12:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
23:12:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
23:12:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
23:12:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:12:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:12:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:31.484736 20957 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
23:12:31.545 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 46e16c4794074d72b0a630bea06b8440
23:12:31.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.572577 21407 tablet_service.cc:1467] Processing CreateTablet for tablet 43b7940ad4144b84b30a6209edd1fe86 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833 [id=46e16c4794074d72b0a630bea06b8440]), partition=RANGE (key) PARTITION UNBOUNDED
23:12:31.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.574772 21407 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 43b7940ad4144b84b30a6209edd1fe86. 1 dirs total, 0 dirs full, 0 dirs failed
23:12:31.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.588415 21266 tablet_service.cc:1467] Processing CreateTablet for tablet 43b7940ad4144b84b30a6209edd1fe86 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833 [id=46e16c4794074d72b0a630bea06b8440]), partition=RANGE (key) PARTITION UNBOUNDED
23:12:31.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.590443 21266 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 43b7940ad4144b84b30a6209edd1fe86. 1 dirs total, 0 dirs full, 0 dirs failed
23:12:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.592149 21125 tablet_service.cc:1467] Processing CreateTablet for tablet 43b7940ad4144b84b30a6209edd1fe86 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833 [id=46e16c4794074d72b0a630bea06b8440]), partition=RANGE (key) PARTITION UNBOUNDED
23:12:31.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.593781 21125 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 43b7940ad4144b84b30a6209edd1fe86. 1 dirs total, 0 dirs full, 0 dirs failed
23:12:31.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.602979 21506 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Bootstrap starting.
23:12:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.610283 21506 tablet_bootstrap.cc:654] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Neither blocks nor log segments found. Creating new log.
23:12:31.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.612526 21506 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Log is configured to *not* fsync() on all Append() calls
23:12:31.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.618505 21508 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap starting.
23:12:31.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.622074 21506 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: No bootstrap required, opened a new log
23:12:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.622905 21506 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Time spent bootstrapping tablet: real 0.020s	user 0.012s	sys 0.007s
23:12:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.623186 21509 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Bootstrap starting.
23:12:31.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.626363 21508 tablet_bootstrap.cc:654] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Neither blocks nor log segments found. Creating new log.
23:12:31.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.628736 21508 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Log is configured to *not* fsync() on all Append() calls
23:12:31.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.630674 21509 tablet_bootstrap.cc:654] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Neither blocks nor log segments found. Creating new log.
23:12:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.632822 21509 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Log is configured to *not* fsync() on all Append() calls
23:12:31.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.634501 21508 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: No bootstrap required, opened a new log
23:12:31.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.635156 21508 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Time spent bootstrapping tablet: real 0.017s	user 0.006s	sys 0.009s
23:12:31.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.644243 21509 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: No bootstrap required, opened a new log
23:12:31.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.644793 21509 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Time spent bootstrapping tablet: real 0.023s	user 0.011s	sys 0.007s
23:12:31.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.648837 21506 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.649780 21506 raft_consensus.cc:374] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:12:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.650081 21506 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c75eda9306bf472caf0f597e71854313, State: Initialized, Role: FOLLOWER
23:12:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.651005 21506 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.661146 21506 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Time spent starting tablet: real 0.038s	user 0.033s	sys 0.001s
23:12:31.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.666674 21508 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:31.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.667601 21508 raft_consensus.cc:374] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:12:31.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.667884 21508 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4da2752128f4adba064520c15942e13, State: Initialized, Role: FOLLOWER
23:12:31.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.668781 21508 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:31.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:31.676191 21477 tablet.cc:2369] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:12:31.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.683554 21508 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Time spent starting tablet: real 0.048s	user 0.041s	sys 0.007s
23:12:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.695832 21509 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.696544 21509 raft_consensus.cc:374] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:12:31.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.696888 21509 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d65993ec41674e8f976e3ecbb62149e4, State: Initialized, Role: FOLLOWER
23:12:31.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.697671 21509 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:31.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.700788 21509 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Time spent starting tablet: real 0.056s	user 0.027s	sys 0.008s
23:12:31.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:31.753723 21337 tablet.cc:2369] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:12:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:31.807273 21194 tablet.cc:2369] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:12:31.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:31.846251 21468 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 21343
23:12:31.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.956291 21513 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:12:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.956758 21513 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:31.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.959091 21513 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.977164 21145 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4" is_pre_election: true
23:12:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.977849 21427 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313" is_pre_election: true
23:12:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.978111 21145 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4da2752128f4adba064520c15942e13 in term 0.
23:12:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.978787 21427 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4da2752128f4adba064520c15942e13 in term 0.
23:12:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.979810 21222 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4; no voters: 
23:12:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.980758 21513 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:12:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.981158 21513 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:12:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.981482 21513 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 0 FOLLOWER]: Advancing to term 1
23:12:31.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.988660 21513 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:31.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.990741 21513 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 1 election: Requested vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:31.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.991633 21427 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313"
23:12:31.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.991966 21145 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4"
23:12:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.992244 21427 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 0 FOLLOWER]: Advancing to term 1
23:12:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.992566 21145 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 0 FOLLOWER]: Advancing to term 1
23:12:31.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.998816 21145 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4da2752128f4adba064520c15942e13 in term 1.
23:12:31.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.999395 21427 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4da2752128f4adba064520c15942e13 in term 1.
23:12:32.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:31.999822 21222 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4; no voters: 
23:12:32.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.000792 21513 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 1 FOLLOWER]: Leader election won for term 1
23:12:32.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.002740 21513 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 1 LEADER]: Becoming Leader. State: Replica: b4da2752128f4adba064520c15942e13, State: Running, Role: LEADER
23:12:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.003684 21513 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.018639 20957 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 reported cstate change: term changed from 0 to 1, leader changed from <none> to b4da2752128f4adba064520c15942e13 (127.20.69.66). New cstate: current_term: 1 leader_uuid: "b4da2752128f4adba064520c15942e13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: HEALTHY } } }
23:12:32.047 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 46e16c4794074d72b0a630bea06b8440
23:12:32.071 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:32.078 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299992, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:32.080 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [13ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=null) after lookup
23:12:32.091 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:32.093 [DEBUG - kudu-nio-1] (Connection.java:757) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] Successfully connected to peer
23:12:32.170 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x3a1fa8ad, L:/127.0.0.1:52298 - R:/127.20.69.66:46673] using TOKEN/null
23:12:32.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.187690 21145 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 1 FOLLOWER]: Refusing update from remote peer b4da2752128f4adba064520c15942e13: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:12:32.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.188678 21427 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 1 FOLLOWER]: Refusing update from remote peer b4da2752128f4adba064520c15942e13: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:12:32.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.189409 21513 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:12:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.190553 21517 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:12:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.226189 21521 mvcc.cc:205] Tried to move back new op lower bound from 7305076540144144384 to 7305076539417210880. Current Snapshot: MvccSnapshot[applied={T|T < 7305076540144144384}]
23:12:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.235702 21523 mvcc.cc:205] Tried to move back new op lower bound from 7305076540144144384 to 7305076539417210880. Current Snapshot: MvccSnapshot[applied={T|T < 7305076540144144384}]
23:12:32.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.262176 21525 mvcc.cc:205] Tried to move back new op lower bound from 7305076540144144384 to 7305076539417210880. Current Snapshot: MvccSnapshot[applied={T|T < 7305076540144144384}]
23:12:32.382 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:12:32.384 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.69.66:46673
23:12:32.386 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:12:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:32.386987 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 21198
23:12:32.415 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:32.418 [DEBUG - kudu-nio-1] (Connection.java:239) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] handling channelInactive
23:12:32.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state READY due to: connection closed
23:12:32.420 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: connection closed
23:12:32.439 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:32.440 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:32.441 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [4ms] received response from server b4da2752128f4adba064520c15942e13: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1883047159(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@ac56092 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@92f8106 -> wakeup thread Time-limited test)) after lookup
23:12:32.441 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:32.443 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.443 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.443 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.459 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:32.459 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:32.460 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [4ms] received response from server b4da2752128f4adba064520c15942e13: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server b4da2752128f4adba064520c15942e13, [27ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [27ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1883047159(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@ac56092 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@92f8106 -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735))) after lookup
23:12:32.462 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:32.464 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.464 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.464 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.498 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:32.499 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:32.500 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=83), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [4ms] received response from server b4da2752128f4adba064520c15942e13: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server b4da2752128f4adba064520c15942e13, [27ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [27ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [45ms] sending RPC to server b4da2752128f4adba064520c15942e13, [48ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [48ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1883047159(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@ac56092 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735) -> (continuation of Deferred@1332865862 after retry RPC@1065712016), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@92f8106 -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735) -> (continuation of Deferred@1332865862 after retry RPC@1065712016))) after lookup
23:12:32.501 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:32.502 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.503 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.503 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.600 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:32.600 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:32.602 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=185), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [4ms] received response from server b4da2752128f4adba064520c15942e13: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server b4da2752128f4adba064520c15942e13, [27ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [27ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [45ms] sending RPC to server b4da2752128f4adba064520c15942e13, [48ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [48ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [85ms] sending RPC to server b4da2752128f4adba064520c15942e13, [87ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [87ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [183ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1883047159(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@ac56092 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735) -> (continuation of Deferred@1332865862 after retry RPC@1065712016) -> (continuation of Deferred@290188397 after retry RPC@1117760840), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@92f8106 -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735) -> (continuation of Deferred@1332865862 after retry RPC@1065712016) -> (continuation of Deferred@290188397 after retry RPC@1117760840))) after lookup
23:12:32.603 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:32.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:32.605 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:33.119 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:33.120 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:33.124 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=707), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [4ms] received response from server b4da2752128f4adba064520c15942e13: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server b4da2752128f4adba064520c15942e13, [27ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [27ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [45ms] sending RPC to server b4da2752128f4adba064520c15942e13, [48ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [48ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [85ms] sending RPC to server b4da2752128f4adba064520c15942e13, [87ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [87ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [183ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [186ms] sending RPC to server b4da2752128f4adba064520c15942e13, [189ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [189ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [697ms] refreshing cache from master, [697ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [702ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1883047159(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@ac56092 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735) -> (continuation of Deferred@1332865862 after retry RPC@1065712016) -> (continuation of Deferred@290188397 after retry RPC@1117760840) -> (continuation of Deferred@1969610237 after retry RPC@1253034996), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@92f8106 -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735) -> (continuation of Deferred@1332865862 after retry RPC@1065712016) -> (continuation of Deferred@290188397 after retry RPC@1117760840) -> (continuation of Deferred@1969610237 after retry RPC@1253034996))) after lookup
23:12:33.125 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:33.126 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:33.127 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:33.127 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:33.739 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:33.740 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:33.741 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1324), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [4ms] received response from server b4da2752128f4adba064520c15942e13: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server b4da2752128f4adba064520c15942e13, [27ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [27ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [45ms] sending RPC to server b4da2752128f4adba064520c15942e13, [48ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [48ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [85ms] sending RPC to server b4da2752128f4adba064520c15942e13, [87ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [87ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [183ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [186ms] sending RPC to server b4da2752128f4adba064520c15942e13, [189ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [189ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [697ms] refreshing cache from master, [697ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [702ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [708ms] sending RPC to server b4da2752128f4adba064520c15942e13, [711ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [711ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [1318ms] refreshing cache from master, [1318ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [1322ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1883047159(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@ac56092 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735) -> (continuation of Deferred@1332865862 after retry RPC@1065712016) -> (continuation of Deferred@290188397 after retry RPC@1117760840) -> (continuation of Deferred@1969610237 after retry RPC@1253034996) -> (continuation of Deferred@109162102 after retry RPC@772288519), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@92f8106 -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735) -> (continuation of Deferred@1332865862 after retry RPC@1065712016) -> (continuation of Deferred@290188397 after retry RPC@1117760840) -> (continuation of Deferred@1969610237 after retry RPC@1253034996) -> (continuation of Deferred@109162102 after retry RPC@772288519))) after lookup
23:12:33.742 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:33.744 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:33.744 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:33.744 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.830775 21533 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader b4da2752128f4adba064520c15942e13)
23:12:33.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.831511 21533 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:33.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.832576 21534 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader b4da2752128f4adba064520c15942e13)
23:12:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.833206 21534 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:33.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.835520 21533 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:33.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.850925 21534 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:33.856421 21360 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:12:33.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:33.863078 21360 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:12:33.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:33.871917 21078 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:12:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.875360 21144 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "c75eda9306bf472caf0f597e71854313" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4" is_pre_election: true
23:12:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.876226 21144 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c75eda9306bf472caf0f597e71854313 in term 1.
23:12:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:33.877326 21078 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:12:33.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.877735 21362 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c75eda9306bf472caf0f597e71854313, d65993ec41674e8f976e3ecbb62149e4; no voters: b4da2752128f4adba064520c15942e13
23:12:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.878680 21534 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 1 FOLLOWER]: Leader pre-election won for term 2
23:12:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.879113 21534 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 1 FOLLOWER]: Starting leader election (detected failure of leader b4da2752128f4adba064520c15942e13)
23:12:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.879462 21534 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 1 FOLLOWER]: Advancing to term 2
23:12:33.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.880894 21427 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "d65993ec41674e8f976e3ecbb62149e4" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313" is_pre_election: true
23:12:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.884658 21534 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.885725 21427 raft_consensus.cc:2379] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d65993ec41674e8f976e3ecbb62149e4 in current term 2: Already voted for candidate c75eda9306bf472caf0f597e71854313 in this term.
23:12:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.887132 21534 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 2 election: Requested vote from peers d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.887388 21144 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "c75eda9306bf472caf0f597e71854313" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4"
23:12:33.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.887943 21144 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 1 FOLLOWER]: Advancing to term 2
23:12:33.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.889385 21078 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d65993ec41674e8f976e3ecbb62149e4; no voters: b4da2752128f4adba064520c15942e13, c75eda9306bf472caf0f597e71854313
23:12:33.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:33.890691 21360 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:12:33.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.893200 21144 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c75eda9306bf472caf0f597e71854313 in term 2.
23:12:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.893646 21533 raft_consensus.cc:2726] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
23:12:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.894152 21362 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c75eda9306bf472caf0f597e71854313, d65993ec41674e8f976e3ecbb62149e4; no voters: b4da2752128f4adba064520c15942e13
23:12:33.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.894972 21534 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 FOLLOWER]: Leader election won for term 2
23:12:33.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.897200 21534 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 LEADER]: Becoming Leader. State: Replica: c75eda9306bf472caf0f597e71854313, State: Running, Role: LEADER
23:12:33.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.898294 21534 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:33.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:33.906596 20957 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 reported cstate change: term changed from 1 to 2, leader changed from b4da2752128f4adba064520c15942e13 (127.20.69.66) to c75eda9306bf472caf0f597e71854313 (127.20.69.67). New cstate: current_term: 2 leader_uuid: "c75eda9306bf472caf0f597e71854313" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: UNKNOWN } } }
23:12:34.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:34.304894 21360 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 -> Peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Couldn't send request to peer b4da2752128f4adba064520c15942e13. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:12:34.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:34.498778 21144 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 2 FOLLOWER]: Refusing update from remote peer c75eda9306bf472caf0f597e71854313: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
23:12:34.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:34.500111 21534 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
23:12:34.839 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:34.839 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:34.840 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2423), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [4ms] received response from server b4da2752128f4adba064520c15942e13: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server b4da2752128f4adba064520c15942e13, [27ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [27ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [45ms] sending RPC to server b4da2752128f4adba064520c15942e13, [48ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [48ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [85ms] sending RPC to server b4da2752128f4adba064520c15942e13, [87ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [87ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [183ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [186ms] sending RPC to server b4da2752128f4adba064520c15942e13, [189ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [189ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [697ms] refreshing cache from master, [697ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [702ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [708ms] sending RPC to server b4da2752128f4adba064520c15942e13, [711ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [711ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [1318ms] refreshing cache from master, [1318ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [1322ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [1326ms] sending RPC to server b4da2752128f4adba064520c15942e13, [1328ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [1329ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [2417ms] refreshing cache from master, [2418ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [2422ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1883047159(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@ac56092 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735) -> (continuation of Deferred@1332865862 after retry RPC@1065712016) -> (continuation of Deferred@290188397 after retry RPC@1117760840) -> (continuation of Deferred@1969610237 after retry RPC@1253034996) -> (continuation of Deferred@109162102 after retry RPC@772288519) -> (continuation of Deferred@901777704 after retry RPC@1520934815), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@92f8106 -> wakeup thread Time-limited test -> (continuation of Deferred@710392299 after retry RPC@968439735) -> (continuation of Deferred@1332865862 after retry RPC@1065712016) -> (continuation of Deferred@290188397 after retry RPC@1117760840) -> (continuation of Deferred@1969610237 after retry RPC@1253034996) -> (continuation of Deferred@109162102 after retry RPC@772288519) -> (continuation of Deferred@901777704 after retry RPC@1520934815))) after lookup
23:12:34.842 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:34.846 [DEBUG - kudu-nio-0] (Connection.java:757) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] Successfully connected to peer
23:12:34.881 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xfa511153, L:/127.0.0.1:43166 - R:/127.20.69.67:43603] using TOKEN/null
23:12:34.927 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.69.66:46673
23:12:34.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:12:34.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:34.930625 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/logs
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.66:46673
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.66
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37625
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.66
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:34.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:12:34.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:34.976502 20908 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1f00 after lost signal to thread 20843
23:12:34.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:34.977258 20908 debug-util.cc:398] Leaking SignalData structure 0x7b080009bdc0 after lost signal to thread 20911
23:12:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:35.276247 21567 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:35.276722 21567 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:35.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:35.277257 21567 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:35.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:35.311929 21567 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:35.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:35.312758 21567 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.66
23:12:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:35.349195 21567 tablet_server_runner.cc:78] Tablet server non-default flags:
23:12:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.66:46673
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.66
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37625
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.66
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/logs
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:35.350569 21567 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:35.352329 21567 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:35.374336 21574 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:36.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:35.375811 21576 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:36.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:35.376010 21573 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:36.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:35.378192 21567 server_base.cc:1034] running on GCE node
23:12:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.708180 21567 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.711103 21567 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:36.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.712532 21567 hybrid_clock.cc:648] HybridClock initialized: now 1783465956712445 us; error 97 us; skew 500 ppm
23:12:36.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.713300 21567 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.725910 21567 webserver.cc:460] Webserver started at http://127.20.69.66:37625/ using document root <none> and password file <none>
23:12:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.726914 21567 fs_manager.cc:359] Metadata directory not provided
23:12:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.727188 21567 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.736380 21567 fs_manager.cc:709] Time spent opening directory manager: real 0.006s	user 0.005s	sys 0.000s
23:12:36.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.742904 21585 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.749812 21567 fs_manager.cc:725] Time spent opening block manager: real 0.010s	user 0.001s	sys 0.003s
23:12:36.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.750227 21567 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:36.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4da2752128f4adba064520c15942e13"
23:12:36.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:28 on dist-test-slave-l596"
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.753235 21567 fs_report.cc:383] FS layout report
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/data
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:36.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.795507 21567 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:36.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.797303 21567 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:36.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.797859 21567 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.801205 21567 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:12:36.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.807446 21592 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:12:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.818614 21567 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:12:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.818913 21567 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.013s	user 0.002s	sys 0.000s
23:12:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.819222 21567 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:12:36.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.826735 21567 ts_tablet_manager.cc:610] Registered 1 tablets
23:12:36.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.826992 21567 ts_tablet_manager.cc:589] Time spent register tablets: real 0.008s	user 0.007s	sys 0.000s
23:12:36.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.827658 21592 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap starting.
23:12:36.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:36.858880 21360 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 -> Peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Couldn't send request to peer b4da2752128f4adba064520c15942e13. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:12:36.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.880921 21592 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Log is configured to *not* fsync() on all Append() calls
23:12:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.915459 21592 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:12:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.916644 21592 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap complete.
23:12:36.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.918782 21592 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Time spent bootstrapping tablet: real 0.092s	user 0.049s	sys 0.038s
23:12:36.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.943763 21592 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:36.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.946406 21592 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4da2752128f4adba064520c15942e13, State: Initialized, Role: FOLLOWER
23:12:36.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.947574 21592 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:36.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:36.952179 21592 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Time spent starting tablet: real 0.033s	user 0.021s	sys 0.013s
23:12:37.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.006510 21567 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.66:46673
23:12:37.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.006718 21702 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.66:46673 every 8 connection(s)
23:12:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.008876 21567 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:12:37.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.011890 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 21567
23:12:37.013 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:37.049 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:12:37.051 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.69.67:43603
23:12:37.051 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:12:37.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.051752 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 21342
23:12:37.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.053958 21705 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:12:37.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.054441 21705 heartbeater.cc:464] Registering TS with master...
23:12:37.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.055652 21705 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:12:37.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.057862 21703 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:12:37.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.058215 21703 heartbeater.cc:464] Registering TS with master...
23:12:37.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.059042 21703 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:12:37.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.062165 20957 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:37.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.062215 20802 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:37.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.062578 21704 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:12:37.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.062893 21704 heartbeater.cc:464] Registering TS with master...
23:12:37.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.063724 21704 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:12:37.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.065549 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.66:57251
23:12:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:37.067209 20877 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:37.089 [DEBUG - kudu-nio-0] (Connection.java:239) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] handling channelInactive
23:12:37.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state READY due to: connection closed
23:12:37.090 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:37.091 [DEBUG - Time-limited test] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:37.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.093 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.093 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.118 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:37.118 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:37.119 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1735208151(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@791a989a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@306b7ce4 -> wakeup thread Time-limited test)) after lookup
23:12:37.119 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:37.120 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.121 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.121 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.139 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:37.139 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:37.140 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [29ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [31ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [31ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [48ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1735208151(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@791a989a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@306b7ce4 -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235))) after lookup
23:12:37.140 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:37.142 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.142 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.143 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.179 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:37.179 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:37.180 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=89), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [29ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [31ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [31ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [48ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [50ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [53ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [53ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [88ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1735208151(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@791a989a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235) -> (continuation of Deferred@1557218732 after retry RPC@2017230484), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@306b7ce4 -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235) -> (continuation of Deferred@1557218732 after retry RPC@2017230484))) after lookup
23:12:37.181 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:37.182 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.182 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.182 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.259 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:37.259 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:37.260 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=169), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [29ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [31ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [31ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [48ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [50ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [53ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [53ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [88ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [90ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [92ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [92ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [168ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1735208151(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@791a989a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235) -> (continuation of Deferred@1557218732 after retry RPC@2017230484) -> (continuation of Deferred@990175043 after retry RPC@1332476113), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@306b7ce4 -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235) -> (continuation of Deferred@1557218732 after retry RPC@2017230484) -> (continuation of Deferred@990175043 after retry RPC@1332476113))) after lookup
23:12:37.261 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:37.263 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.264 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.264 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.758 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:37.758 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:37.759 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=668), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [29ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [31ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [31ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [48ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [50ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [53ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [53ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [88ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [90ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [92ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [92ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [168ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [171ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [174ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [174ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [663ms] refreshing cache from master, [663ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [667ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1735208151(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@791a989a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235) -> (continuation of Deferred@1557218732 after retry RPC@2017230484) -> (continuation of Deferred@990175043 after retry RPC@1332476113) -> (continuation of Deferred@1098785362 after retry RPC@344132826), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@306b7ce4 -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235) -> (continuation of Deferred@1557218732 after retry RPC@2017230484) -> (continuation of Deferred@990175043 after retry RPC@1332476113) -> (continuation of Deferred@1098785362 after retry RPC@344132826))) after lookup
23:12:37.760 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:37.761 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.761 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:37.761 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:38.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.070055 21703 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:12:38.399 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:38.400 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:38.401 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1310), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [29ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [31ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [31ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [48ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [50ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [53ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [53ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [88ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [90ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [92ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [92ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [168ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [171ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [174ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [174ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [663ms] refreshing cache from master, [663ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [667ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [669ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [671ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [671ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [1304ms] refreshing cache from master, [1304ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [1308ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1735208151(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@791a989a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235) -> (continuation of Deferred@1557218732 after retry RPC@2017230484) -> (continuation of Deferred@990175043 after retry RPC@1332476113) -> (continuation of Deferred@1098785362 after retry RPC@344132826) -> (continuation of Deferred@1251796004 after retry RPC@2021355139), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@306b7ce4 -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235) -> (continuation of Deferred@1557218732 after retry RPC@2017230484) -> (continuation of Deferred@990175043 after retry RPC@1332476113) -> (continuation of Deferred@1098785362 after retry RPC@344132826) -> (continuation of Deferred@1251796004 after retry RPC@2021355139))) after lookup
23:12:38.402 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:38.403 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:38.403 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:38.404 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:38.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.484781 21714 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader c75eda9306bf472caf0f597e71854313)
23:12:38.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.485188 21714 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:38.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.486627 21714 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:38.492408 21078 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111)
23:12:38.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.503077 21658 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "d65993ec41674e8f976e3ecbb62149e4" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "b4da2752128f4adba064520c15942e13" is_pre_election: true
23:12:38.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.503862 21658 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d65993ec41674e8f976e3ecbb62149e4 in term 1.
23:12:38.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.505123 21078 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4; no voters: c75eda9306bf472caf0f597e71854313
23:12:38.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.505832 21714 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 2 FOLLOWER]: Leader pre-election won for term 3
23:12:38.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.506142 21714 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 2 FOLLOWER]: Starting leader election (detected failure of leader c75eda9306bf472caf0f597e71854313)
23:12:38.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.506363 21714 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 2 FOLLOWER]: Advancing to term 3
23:12:38.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.510511 21714 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:38.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.512063 21714 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 3 election: Requested vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.513672 21658 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "d65993ec41674e8f976e3ecbb62149e4" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "b4da2752128f4adba064520c15942e13"
23:12:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.514223 21658 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 1 FOLLOWER]: Advancing to term 3
23:12:38.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:38.515044 21078 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111)
23:12:38.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.520954 21658 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d65993ec41674e8f976e3ecbb62149e4 in term 3.
23:12:38.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.521801 21078 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4; no voters: c75eda9306bf472caf0f597e71854313
23:12:38.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.522400 21714 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 3 FOLLOWER]: Leader election won for term 3
23:12:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.523427 21714 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 3 LEADER]: Becoming Leader. State: Replica: d65993ec41674e8f976e3ecbb62149e4, State: Running, Role: LEADER
23:12:38.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.524293 21714 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:38.532814 20957 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 reported cstate change: term changed from 2 to 3, leader changed from c75eda9306bf472caf0f597e71854313 (127.20.69.67) to d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65). New cstate: current_term: 3 leader_uuid: "d65993ec41674e8f976e3ecbb62149e4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: UNKNOWN } } }
23:12:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:39.069427 21658 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 3 FOLLOWER]: Refusing update from remote peer d65993ec41674e8f976e3ecbb62149e4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
23:12:39.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:39.071277 21732 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
23:12:39.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:39.111824 21078 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111) [suppressed 2 similar messages]
23:12:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:39.115350 21078 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 -> Peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Couldn't send request to peer c75eda9306bf472caf0f597e71854313. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:12:41.121 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:41.121 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:41.122 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=4031), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [29ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [31ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [31ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [48ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [50ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [53ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [53ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [88ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [90ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [92ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [92ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [168ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [171ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [174ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [174ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [663ms] refreshing cache from master, [663ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [667ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [669ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [671ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [671ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [1304ms] refreshing cache from master, [1304ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [1308ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [1311ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [1314ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [1314ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [4023ms] refreshing cache from master, [4024ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [4029ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1735208151(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@791a989a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235) -> (continuation of Deferred@1557218732 after retry RPC@2017230484) -> (continuation of Deferred@990175043 after retry RPC@1332476113) -> (continuation of Deferred@1098785362 after retry RPC@344132826) -> (continuation of Deferred@1251796004 after retry RPC@2021355139) -> (continuation of Deferred@271530209 after retry RPC@864240401), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@306b7ce4 -> wakeup thread Time-limited test -> (continuation of Deferred@138908741 after retry RPC@1036198235) -> (continuation of Deferred@1557218732 after retry RPC@2017230484) -> (continuation of Deferred@990175043 after retry RPC@1332476113) -> (continuation of Deferred@1098785362 after retry RPC@344132826) -> (continuation of Deferred@1251796004 after retry RPC@2021355139) -> (continuation of Deferred@271530209 after retry RPC@864240401))) after lookup
23:12:41.123 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:41.124 [DEBUG - kudu-nio-0] (Connection.java:757) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] Successfully connected to peer
23:12:41.151 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd9c67aba, L:/127.0.0.1:60002 - R:/127.20.69.65:45117] using TOKEN/null
23:12:41.236 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.69.67:43603
23:12:41.237 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:12:41.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:41.238554 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data
23:12:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/logs
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.67:43603
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.67
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39541
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.67
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:12:41.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:41.580734 21760 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:41.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:41.581300 21760 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:41.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:41.581939 21760 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:41.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:41.613540 21760 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:41.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:41.614639 21760 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.67
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:41.649796 21760 tablet_server_runner.cc:78] Tablet server non-default flags:
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.67:43603
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.67
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39541
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.67
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/logs
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:41.651306 21760 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:41.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:41.653007 21760 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:41.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:41.673437 21768 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:41.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:41.851070 21078 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 -> Peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Couldn't send request to peer c75eda9306bf472caf0f597e71854313. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:12:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:41.673487 21771 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:42.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:41.675443 21760 server_base.cc:1034] running on GCE node
23:12:42.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:41.673619 21769 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:42.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.889789 21760 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:42.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.892020 21760 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.893384 21760 hybrid_clock.cc:648] HybridClock initialized: now 1783465962893357 us; error 51 us; skew 500 ppm
23:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.894346 21760 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.905025 21760 webserver.cc:460] Webserver started at http://127.20.69.67:39541/ using document root <none> and password file <none>
23:12:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.906046 21760 fs_manager.cc:359] Metadata directory not provided
23:12:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.906260 21760 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.914469 21760 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.003s	sys 0.001s
23:12:42.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.919328 21780 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.920688 21760 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
23:12:42.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.920996 21760 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:42.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c75eda9306bf472caf0f597e71854313"
23:12:42.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:30 on dist-test-slave-l596"
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.923651 21760 fs_report.cc:383] FS layout report
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/data
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.984668 21760 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:42.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.986281 21760 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:42.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.986733 21760 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.989377 21760 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:12:42.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:42.995287 21787 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:12:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.004427 21760 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:12:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.004662 21760 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
23:12:43.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.004873 21760 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:12:43.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.010231 21760 ts_tablet_manager.cc:610] Registered 1 tablets
23:12:43.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.010495 21760 ts_tablet_manager.cc:589] Time spent register tablets: real 0.006s	user 0.002s	sys 0.003s
23:12:43.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.010879 21787 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Bootstrap starting.
23:12:43.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.086467 21787 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Log is configured to *not* fsync() on all Append() calls
23:12:43.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.138275 21787 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:12:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.139884 21787 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Bootstrap complete.
23:12:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.143965 21787 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Time spent bootstrapping tablet: real 0.134s	user 0.085s	sys 0.041s
23:12:43.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.176548 21787 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:43.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.181077 21787 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: c75eda9306bf472caf0f597e71854313, State: Initialized, Role: FOLLOWER
23:12:43.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.182302 21787 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:43.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.188164 21787 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Time spent starting tablet: real 0.044s	user 0.025s	sys 0.016s
23:12:43.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.235108 21760 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.67:43603
23:12:43.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.235324 21896 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.67:43603 every 8 connection(s)
23:12:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.237847 21760 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:12:43.247 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:43.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.246246 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 21760
23:12:43.287 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:12:43.287 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.69.65:45117
23:12:43.288 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:12:43.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.288743 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 21036
23:12:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.294840 21897 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:12:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.295301 21897 heartbeater.cc:464] Registering TS with master...
23:12:43.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.296542 21897 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:12:43.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.299454 21898 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:12:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.299808 21898 heartbeater.cc:464] Registering TS with master...
23:12:43.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.300308 21899 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:12:43.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.300637 21899 heartbeater.cc:464] Registering TS with master...
23:12:43.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.300668 21898 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:12:43.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.300766 20957 ts_manager.cc:194] Re-registered known tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:12:43.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.301450 21899 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:12:43.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.304291 20877 ts_manager.cc:194] Re-registered known tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:12:43.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.304625 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.67:37423
23:12:43.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:43.305485 20802 ts_manager.cc:194] Re-registered known tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:12:43.324 [DEBUG - kudu-nio-0] (Connection.java:239) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] handling channelInactive
23:12:43.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state READY due to: connection closed
23:12:43.325 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:43.326 [DEBUG - Time-limited test] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:43.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:43.326777 21908 negotiation.cc:337] Failed RPC negotiation. Trace:
23:12:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:12:43.288542 (+     0us) reactor.cc:625] Submitting negotiation task for server connection from 127.20.69.65:36961 (local address 127.20.69.67:43603)
23:12:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:12:43.291510 (+  2968us) server_negotiation.cc:206] Beginning negotiation
23:12:43.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:12:43.291543 (+    33us) server_negotiation.cc:399] Waiting for connection header
23:12:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:12:43.324899 (+ 33356us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.20.69.65:36961: BlockingRecv error: recv got EOF from 127.20.69.65:36961 (error 108)
23:12:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2336,"thread_start_us":2917,"threads_started":1}
23:12:43.329 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.337 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:43.337 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:43.338 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [3ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@391993004(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@13fe453b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c26c7ff -> wakeup thread Time-limited test)) after lookup
23:12:43.338 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:43.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.339 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.357 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:43.357 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:43.358 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [3ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [12ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [13ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [13ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@391993004(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@13fe453b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c26c7ff -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995))) after lookup
23:12:43.358 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:43.360 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.360 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.360 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:43.398 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:43.400 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [3ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [12ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [13ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [13ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [32ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [34ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [34ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@391993004(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@13fe453b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995) -> (continuation of Deferred@736730171 after retry RPC@549440251), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c26c7ff -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995) -> (continuation of Deferred@736730171 after retry RPC@549440251))) after lookup
23:12:43.401 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:43.402 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.402 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.402 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.538 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:43.538 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:43.538 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=212), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [3ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [12ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [13ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [13ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [32ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [34ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [34ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [74ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [76ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [76ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [211ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@391993004(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@13fe453b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995) -> (continuation of Deferred@736730171 after retry RPC@549440251) -> (continuation of Deferred@824018654 after retry RPC@2072119344), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c26c7ff -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995) -> (continuation of Deferred@736730171 after retry RPC@549440251) -> (continuation of Deferred@824018654 after retry RPC@2072119344))) after lookup
23:12:43.540 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:43.541 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.541 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.541 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.718 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:43.719 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:43.720 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=393), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [3ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [12ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [13ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [13ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [32ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [34ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [34ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [74ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [76ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [76ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [211ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [213ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [215ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [216ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [387ms] refreshing cache from master, [387ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [392ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@391993004(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@13fe453b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995) -> (continuation of Deferred@736730171 after retry RPC@549440251) -> (continuation of Deferred@824018654 after retry RPC@2072119344) -> (continuation of Deferred@1329530189 after retry RPC@613893906), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c26c7ff -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995) -> (continuation of Deferred@736730171 after retry RPC@549440251) -> (continuation of Deferred@824018654 after retry RPC@2072119344) -> (continuation of Deferred@1329530189 after retry RPC@613893906))) after lookup
23:12:43.720 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:43.722 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.722 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:43.722 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:44.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.309345 21897 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:12:44.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.423363 21909 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:12:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.423823 21909 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:44.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.426407 21909 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:44.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:44.430433 21784 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111)
23:12:44.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:44.433890 21784 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111)
23:12:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.439806 21658 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "c75eda9306bf472caf0f597e71854313" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "b4da2752128f4adba064520c15942e13" is_pre_election: true
23:12:44.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.441192 21782 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c75eda9306bf472caf0f597e71854313; no voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4
23:12:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.442009 21909 raft_consensus.cc:2726] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
23:12:44.497 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:44.497 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:44.498 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1172), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [3ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [12ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [13ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [13ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [32ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [34ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [34ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [74ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [76ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [76ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [211ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [213ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [215ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [216ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [387ms] refreshing cache from master, [387ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [392ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [394ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [396ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [396ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [1167ms] refreshing cache from master, [1167ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [1171ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@391993004(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@13fe453b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995) -> (continuation of Deferred@736730171 after retry RPC@549440251) -> (continuation of Deferred@824018654 after retry RPC@2072119344) -> (continuation of Deferred@1329530189 after retry RPC@613893906) -> (continuation of Deferred@468278974 after retry RPC@1170104446), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c26c7ff -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995) -> (continuation of Deferred@736730171 after retry RPC@549440251) -> (continuation of Deferred@824018654 after retry RPC@2072119344) -> (continuation of Deferred@1329530189 after retry RPC@613893906) -> (continuation of Deferred@468278974 after retry RPC@1170104446))) after lookup
23:12:44.499 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:44.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:44.501 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:44.501 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:44.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.705068 21914 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader d65993ec41674e8f976e3ecbb62149e4)
23:12:44.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.705474 21914 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:44.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.707890 21914 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:44.710955 21589 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111)
23:12:44.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:44.715155 21589 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111)
23:12:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.721617 21851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313" is_pre_election: true
23:12:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.722273 21851 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4da2752128f4adba064520c15942e13 in term 2.
23:12:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.723639 21587 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, c75eda9306bf472caf0f597e71854313; no voters: d65993ec41674e8f976e3ecbb62149e4
23:12:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.724395 21914 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 3 FOLLOWER]: Leader pre-election won for term 4
23:12:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.724704 21914 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 3 FOLLOWER]: Starting leader election (detected failure of leader d65993ec41674e8f976e3ecbb62149e4)
23:12:44.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.724956 21914 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 3 FOLLOWER]: Advancing to term 4
23:12:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.729341 21914 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.730949 21914 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 4 election: Requested vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:44.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.731746 21851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313"
23:12:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.732311 21851 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 2 FOLLOWER]: Advancing to term 4
23:12:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:44.734042 21589 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111)
23:12:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.740209 21851 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4da2752128f4adba064520c15942e13 in term 4.
23:12:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.741079 21587 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, c75eda9306bf472caf0f597e71854313; no voters: d65993ec41674e8f976e3ecbb62149e4
23:12:44.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.741813 21914 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 4 FOLLOWER]: Leader election won for term 4
23:12:44.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.743389 21914 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 4 LEADER]: Becoming Leader. State: Replica: b4da2752128f4adba064520c15942e13, State: Running, Role: LEADER
23:12:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.744277 21914 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:44.753484 20957 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 reported cstate change: term changed from 3 to 4, leader changed from d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65) to b4da2752128f4adba064520c15942e13 (127.20.69.66). New cstate: current_term: 4 leader_uuid: "b4da2752128f4adba064520c15942e13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: HEALTHY } } }
23:12:45.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:45.334151 21851 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 4 FOLLOWER]: Refusing update from remote peer b4da2752128f4adba064520c15942e13: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
23:12:45.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:45.334901 21589 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 -> Peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Couldn't send request to peer d65993ec41674e8f976e3ecbb62149e4. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:12:45.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:45.335830 21933 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.001s
23:12:45.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:45.348050 21937 mvcc.cc:205] Tried to move back new op lower bound from 7305076576896348160 to 7305076566123134976. Current Snapshot: MvccSnapshot[applied={T|T < 7305076576896348160}]
23:12:47.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:47.800100 21589 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 -> Peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Couldn't send request to peer d65993ec41674e8f976e3ecbb62149e4. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:12:48.597 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:48.597 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:48.599 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=5273), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [3ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [12ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [13ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [13ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [32ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [34ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [34ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [74ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [76ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [76ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [211ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [213ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [215ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [216ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [387ms] refreshing cache from master, [387ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [392ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [394ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [396ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [396ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [1167ms] refreshing cache from master, [1167ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [1171ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [1172ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [1175ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [1175ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [5267ms] refreshing cache from master, [5267ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [5270ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@391993004(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@13fe453b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995) -> (continuation of Deferred@736730171 after retry RPC@549440251) -> (continuation of Deferred@824018654 after retry RPC@2072119344) -> (continuation of Deferred@1329530189 after retry RPC@613893906) -> (continuation of Deferred@468278974 after retry RPC@1170104446) -> (continuation of Deferred@20417185 after retry RPC@935114601), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c26c7ff -> wakeup thread Time-limited test -> (continuation of Deferred@1166826312 after retry RPC@1340642995) -> (continuation of Deferred@736730171 after retry RPC@549440251) -> (continuation of Deferred@824018654 after retry RPC@2072119344) -> (continuation of Deferred@1329530189 after retry RPC@613893906) -> (continuation of Deferred@468278974 after retry RPC@1170104446) -> (continuation of Deferred@20417185 after retry RPC@935114601))) after lookup
23:12:48.600 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:48.601 [DEBUG - kudu-nio-0] (Connection.java:757) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] Successfully connected to peer
23:12:48.626 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x08f4cc8f, L:/127.0.0.1:50374 - R:/127.20.69.66:46673] using TOKEN/null
23:12:48.673 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.69.65:45117
23:12:48.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:12:48.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:48.675523 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:48.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:48.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/logs
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.65:45117
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.65
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45683
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.65
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:12:48.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:48.986529 21964 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:48.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:48.987023 21964 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:48.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:48.987444 21964 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:49.026851 21964 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:49.027812 21964 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.65
23:12:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:49.063941 21964 tablet_server_runner.cc:78] Tablet server non-default flags:
23:12:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.65:45117
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.65
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45683
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.65
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/logs
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:49.065469 21964 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:49.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:49.067175 21964 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:49.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:49.086709 21975 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:50.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:50.058746 21589 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111) [suppressed 10 similar messages]
23:12:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:49.088351 21964 server_base.cc:1034] running on GCE node
23:12:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:49.087600 21972 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:49.086921 21973 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:50.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.296864 21964 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:50.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.299328 21964 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:50.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.300779 21964 hybrid_clock.cc:648] HybridClock initialized: now 1783465970300745 us; error 37 us; skew 500 ppm
23:12:50.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.301604 21964 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.311390 21964 webserver.cc:460] Webserver started at http://127.20.69.65:45683/ using document root <none> and password file <none>
23:12:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.312278 21964 fs_manager.cc:359] Metadata directory not provided
23:12:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.312476 21964 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:50.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.323654 21964 fs_manager.cc:709] Time spent opening directory manager: real 0.008s	user 0.002s	sys 0.007s
23:12:50.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.328413 21989 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:50.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.329672 21964 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
23:12:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.329957 21964 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:12:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d65993ec41674e8f976e3ecbb62149e4"
23:12:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:25 on dist-test-slave-l596"
23:12:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.332252 21964 fs_report.cc:383] FS layout report
23:12:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:12:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:12:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/data
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:50.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.374354 21964 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.375767 21964 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.376175 21964 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.378654 21964 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:12:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.384187 21996 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:12:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.391738 21964 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:12:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.392038 21964 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.009s	user 0.001s	sys 0.000s
23:12:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.392356 21964 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:12:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.399883 21964 ts_tablet_manager.cc:610] Registered 1 tablets
23:12:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.400157 21964 ts_tablet_manager.cc:589] Time spent register tablets: real 0.008s	user 0.008s	sys 0.000s
23:12:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.401043 21996 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Bootstrap starting.
23:12:50.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.460186 21996 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Log is configured to *not* fsync() on all Append() calls
23:12:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.518044 21996 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:12:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.519452 21996 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Bootstrap complete.
23:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.521580 21996 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Time spent bootstrapping tablet: real 0.121s	user 0.090s	sys 0.027s
23:12:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:50.548648 21589 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 -> Peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Couldn't send request to peer d65993ec41674e8f976e3ecbb62149e4. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
23:12:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.548914 21996 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.552357 21996 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: d65993ec41674e8f976e3ecbb62149e4, State: Initialized, Role: FOLLOWER
23:12:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.553258 21996 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:50.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.558791 21996 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Time spent starting tablet: real 0.037s	user 0.029s	sys 0.005s
23:12:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.588266 21964 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.65:45117
23:12:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.588487 22104 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.65:45117 every 8 connection(s)
23:12:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.591419 21964 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:12:50.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.601683 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 21964
23:12:50.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:50.628 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:12:50.636 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.69.66:46673
23:12:50.637 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:12:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.637486 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 21567
23:12:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.644347 22105 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:12:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.644913 22105 heartbeater.cc:464] Registering TS with master...
23:12:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.645582 22107 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:12:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.645931 22107 heartbeater.cc:464] Registering TS with master...
23:12:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.646333 22105 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:12:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.647004 22107 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:12:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.647652 22106 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:12:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.647971 22106 heartbeater.cc:464] Registering TS with master...
23:12:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.648797 22106 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:12:50.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.651377 20957 ts_manager.cc:194] Re-registered known tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:50.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.655083 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.65:52687
23:12:50.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.655982 20876 ts_manager.cc:194] Re-registered known tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:50.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:50.656901 20802 ts_manager.cc:194] Re-registered known tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:50.680 [DEBUG - kudu-nio-0] (Connection.java:239) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] handling channelInactive
23:12:50.681 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state READY due to: connection closed
23:12:50.681 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:50.682 [DEBUG - Time-limited test] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:50.684 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.684 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.685 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.697 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:50.698 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:50.698 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@2037593188(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@43a26494 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@484f9d9 -> wakeup thread Time-limited test)) after lookup
23:12:50.699 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:50.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.700 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.717 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:50.717 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:50.717 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [16ms] sending RPC to server b4da2752128f4adba064520c15942e13, [18ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [18ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@2037593188(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@43a26494 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@831657754 after retry RPC@1147093724), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@484f9d9 -> wakeup thread Time-limited test -> (continuation of Deferred@831657754 after retry RPC@1147093724))) after lookup
23:12:50.718 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:50.720 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.720 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.720 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.738 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:50.739 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:50.739 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [16ms] sending RPC to server b4da2752128f4adba064520c15942e13, [18ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [18ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [36ms] sending RPC to server b4da2752128f4adba064520c15942e13, [38ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [38ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [56ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@2037593188(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@43a26494 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@831657754 after retry RPC@1147093724) -> (continuation of Deferred@227267835 after retry RPC@872169012), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@484f9d9 -> wakeup thread Time-limited test -> (continuation of Deferred@831657754 after retry RPC@1147093724) -> (continuation of Deferred@227267835 after retry RPC@872169012))) after lookup
23:12:50.740 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:50.740 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.741 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.818 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:50.819 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:50.819 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=137), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [16ms] sending RPC to server b4da2752128f4adba064520c15942e13, [18ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [18ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [36ms] sending RPC to server b4da2752128f4adba064520c15942e13, [38ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [38ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [56ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [57ms] sending RPC to server b4da2752128f4adba064520c15942e13, [59ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [59ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [136ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@2037593188(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@43a26494 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@831657754 after retry RPC@1147093724) -> (continuation of Deferred@227267835 after retry RPC@872169012) -> (continuation of Deferred@195551905 after retry RPC@151789115), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@484f9d9 -> wakeup thread Time-limited test -> (continuation of Deferred@831657754 after retry RPC@1147093724) -> (continuation of Deferred@227267835 after retry RPC@872169012) -> (continuation of Deferred@195551905 after retry RPC@151789115))) after lookup
23:12:50.820 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:50.824 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.824 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:50.824 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:51.318 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:51.318 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:51.319 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=636), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [16ms] sending RPC to server b4da2752128f4adba064520c15942e13, [18ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [18ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [36ms] sending RPC to server b4da2752128f4adba064520c15942e13, [38ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [38ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [56ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [57ms] sending RPC to server b4da2752128f4adba064520c15942e13, [59ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [59ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [136ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [138ms] sending RPC to server b4da2752128f4adba064520c15942e13, [142ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [142ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [631ms] refreshing cache from master, [631ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [635ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@2037593188(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@43a26494 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@831657754 after retry RPC@1147093724) -> (continuation of Deferred@227267835 after retry RPC@872169012) -> (continuation of Deferred@195551905 after retry RPC@151789115) -> (continuation of Deferred@332600760 after retry RPC@2051484735), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@484f9d9 -> wakeup thread Time-limited test -> (continuation of Deferred@831657754 after retry RPC@1147093724) -> (continuation of Deferred@227267835 after retry RPC@872169012) -> (continuation of Deferred@195551905 after retry RPC@151789115) -> (continuation of Deferred@332600760 after retry RPC@2051484735))) after lookup
23:12:51.320 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:12:51.321 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:51.321 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:51.322 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:12:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.660781 22105 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:12:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.792662 22117 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b4da2752128f4adba064520c15942e13)
23:12:51.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.793310 22117 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.795310 22117 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:51.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:51.808697 21782 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:12:51.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:51.812657 21782 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:12:51.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.821182 22059 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "c75eda9306bf472caf0f597e71854313" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4" is_pre_election: true
23:12:51.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.822160 22059 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c75eda9306bf472caf0f597e71854313 in term 3.
23:12:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.823760 21784 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c75eda9306bf472caf0f597e71854313, d65993ec41674e8f976e3ecbb62149e4; no voters: b4da2752128f4adba064520c15942e13
23:12:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.824577 22117 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 4 FOLLOWER]: Leader pre-election won for term 5
23:12:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.824944 22117 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 4 FOLLOWER]: Starting leader election (detected failure of leader b4da2752128f4adba064520c15942e13)
23:12:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.825311 22117 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 4 FOLLOWER]: Advancing to term 5
23:12:51.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.832757 22117 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.834741 22117 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 5 election: Requested vote from peers d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.835465 22059 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "c75eda9306bf472caf0f597e71854313" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4"
23:12:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.835999 22059 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 3 FOLLOWER]: Advancing to term 5
23:12:51.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:51.839021 21782 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:12:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.845037 22059 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c75eda9306bf472caf0f597e71854313 in term 5.
23:12:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.846101 21784 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c75eda9306bf472caf0f597e71854313, d65993ec41674e8f976e3ecbb62149e4; no voters: b4da2752128f4adba064520c15942e13
23:12:51.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.846922 22117 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 5 FOLLOWER]: Leader election won for term 5
23:12:51.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.848290 22117 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 5 LEADER]: Becoming Leader. State: Replica: c75eda9306bf472caf0f597e71854313, State: Running, Role: LEADER
23:12:51.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.849391 22117 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:51.860491 20957 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 reported cstate change: term changed from 4 to 5, leader changed from b4da2752128f4adba064520c15942e13 (127.20.69.66) to c75eda9306bf472caf0f597e71854313 (127.20.69.67). New cstate: current_term: 5 leader_uuid: "c75eda9306bf472caf0f597e71854313" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: UNKNOWN } } }
23:12:52.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:52.314924 21782 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 -> Peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Couldn't send request to peer b4da2752128f4adba064520c15942e13. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:12:52.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:52.411185 22059 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 5 FOLLOWER]: Refusing update from remote peer c75eda9306bf472caf0f597e71854313: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
23:12:52.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:52.412479 22117 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
23:12:53.138 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:53.138 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:53.139 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2457), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [16ms] sending RPC to server b4da2752128f4adba064520c15942e13, [18ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [18ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [36ms] sending RPC to server b4da2752128f4adba064520c15942e13, [38ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [38ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [56ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [57ms] sending RPC to server b4da2752128f4adba064520c15942e13, [59ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [59ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [136ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [138ms] sending RPC to server b4da2752128f4adba064520c15942e13, [142ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [142ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [631ms] refreshing cache from master, [631ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [635ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [638ms] sending RPC to server b4da2752128f4adba064520c15942e13, [639ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [640ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [2452ms] refreshing cache from master, [2452ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [2456ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@2037593188(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@43a26494 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@831657754 after retry RPC@1147093724) -> (continuation of Deferred@227267835 after retry RPC@872169012) -> (continuation of Deferred@195551905 after retry RPC@151789115) -> (continuation of Deferred@332600760 after retry RPC@2051484735) -> (continuation of Deferred@537970990 after retry RPC@139817666), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@484f9d9 -> wakeup thread Time-limited test -> (continuation of Deferred@831657754 after retry RPC@1147093724) -> (continuation of Deferred@227267835 after retry RPC@872169012) -> (continuation of Deferred@195551905 after retry RPC@151789115) -> (continuation of Deferred@332600760 after retry RPC@2051484735) -> (continuation of Deferred@537970990 after retry RPC@139817666))) after lookup
23:12:53.140 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:53.142 [DEBUG - kudu-nio-1] (Connection.java:757) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] Successfully connected to peer
23:12:53.165 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x1123985e, L:/127.0.0.1:51880 - R:/127.20.69.67:43603] using TOKEN/null
23:12:53.216 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.69.66:46673
23:12:53.216 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:12:53.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:53.217644 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/logs
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.66:46673
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.66
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37625
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.66
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:12:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:53.541810 22151 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:53.542361 22151 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:53.543026 22151 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:53.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:53.580857 22151 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:53.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:53.582105 22151 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.66
23:12:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:53.621007 22151 tablet_server_runner.cc:78] Tablet server non-default flags:
23:12:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data
23:12:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.66:46673
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.66
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37625
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.66
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/logs
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:53.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:53.622575 22151 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:53.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:53.624284 22151 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:53.643827 22160 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:54.812878 21782 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 -> Peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Couldn't send request to peer b4da2752128f4adba064520c15942e13. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:12:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:53.643877 22157 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:53.643885 22158 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:53.645895 22151 server_base.cc:1034] running on GCE node
23:12:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.853955 22151 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.856417 22151 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.857846 22151 hybrid_clock.cc:648] HybridClock initialized: now 1783465974857805 us; error 37 us; skew 500 ppm
23:12:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.858798 22151 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:54.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.865479 22151 webserver.cc:460] Webserver started at http://127.20.69.66:37625/ using document root <none> and password file <none>
23:12:54.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.866639 22151 fs_manager.cc:359] Metadata directory not provided
23:12:54.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.866906 22151 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.876032 22151 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
23:12:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.880532 22173 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:54.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.881588 22151 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
23:12:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.881946 22151 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4da2752128f4adba064520c15942e13"
23:12:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:28 on dist-test-slave-l596"
23:12:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.884302 22151 fs_report.cc:383] FS layout report
23:12:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/data
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:54.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.922257 22151 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:54.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.923818 22151 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:54.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.924324 22151 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.926975 22151 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:12:54.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.933017 22180 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:12:54.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.940943 22151 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:12:54.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.941200 22151 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
23:12:54.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.941587 22151 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:12:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.946996 22151 ts_tablet_manager.cc:610] Registered 1 tablets
23:12:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.947229 22151 ts_tablet_manager.cc:589] Time spent register tablets: real 0.006s	user 0.005s	sys 0.000s
23:12:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:54.947491 22180 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap starting.
23:12:55.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.007609 22180 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Log is configured to *not* fsync() on all Append() calls
23:12:55.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.099931 22180 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:12:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.101629 22180 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap complete.
23:12:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.104252 22180 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Time spent bootstrapping tablet: real 0.157s	user 0.110s	sys 0.044s
23:12:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.125339 22151 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.66:46673
23:12:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.125545 22287 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.66:46673 every 8 connection(s)
23:12:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.129315 22151 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:12:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.133934 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 22151
23:12:55.134 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:55.157 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:12:55.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.140981 22180 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:55.158 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.69.67:43603
23:12:55.158 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:12:55.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.161698 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 21760
23:12:55.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.155225 22180 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4da2752128f4adba064520c15942e13, State: Initialized, Role: FOLLOWER
23:12:55.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.170658 22180 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:55.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.186362 22180 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Time spent starting tablet: real 0.082s	user 0.042s	sys 0.004s
23:12:55.206 [DEBUG - kudu-nio-1] (Connection.java:239) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] handling channelInactive
23:12:55.206 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state READY due to: connection closed
23:12:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.207370 22288 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:12:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.207870 22288 heartbeater.cc:464] Registering TS with master...
23:12:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.208076 22290 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:12:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.208387 22290 heartbeater.cc:464] Registering TS with master...
23:12:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.209141 22288 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:12:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.209323 22290 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:12:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.209458 22289 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:12:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.209841 22289 heartbeater.cc:464] Registering TS with master...
23:12:55.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:55.212 [DEBUG - Time-limited test] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:55.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.210739 22289 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:12:55.216 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.216 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.216 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.215258 20957 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.215481 20802 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.218255 20877 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.218755 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.66:45337
23:12:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:55.223992 22288 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:12:55.237 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:55.237 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:55.238 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [4ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [4ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1147778988(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@32415480 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6260df4e -> wakeup thread Time-limited test)) after lookup
23:12:55.238 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:55.240 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.240 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.240 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.257 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:55.258 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:55.258 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [4ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [4ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [26ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [28ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [28ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [45ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1147778988(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@32415480 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@10569739 after retry RPC@1683975678), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6260df4e -> wakeup thread Time-limited test -> (continuation of Deferred@10569739 after retry RPC@1683975678))) after lookup
23:12:55.259 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:55.260 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.260 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.260 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.298 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:55.318 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:55.320 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=108), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [4ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [4ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [26ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [28ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [28ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [45ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [47ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [48ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [48ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [85ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1147778988(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@32415480 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@10569739 after retry RPC@1683975678) -> (continuation of Deferred@718282510 after retry RPC@1624474368), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6260df4e -> wakeup thread Time-limited test -> (continuation of Deferred@10569739 after retry RPC@1683975678) -> (continuation of Deferred@718282510 after retry RPC@1624474368))) after lookup
23:12:55.321 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:55.323 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.323 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.323 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.457 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:55.457 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:55.458 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=246), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [4ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [4ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [26ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [28ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [28ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [45ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [47ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [48ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [48ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [85ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [108ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [111ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [111ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [241ms] refreshing cache from master, [241ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [244ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1147778988(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@32415480 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@10569739 after retry RPC@1683975678) -> (continuation of Deferred@718282510 after retry RPC@1624474368) -> (continuation of Deferred@687380555 after retry RPC@1422906839), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6260df4e -> wakeup thread Time-limited test -> (continuation of Deferred@10569739 after retry RPC@1683975678) -> (continuation of Deferred@718282510 after retry RPC@1624474368) -> (continuation of Deferred@687380555 after retry RPC@1422906839))) after lookup
23:12:55.460 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:55.462 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.462 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.462 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.617 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:55.617 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:12:55.618 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=406), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [4ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [4ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [26ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [28ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [28ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [45ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [47ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [48ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [48ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [85ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [108ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [111ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [111ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [241ms] refreshing cache from master, [241ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [244ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [247ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [250ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [250ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [401ms] refreshing cache from master, [401ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [404ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1147778988(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@32415480 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@10569739 after retry RPC@1683975678) -> (continuation of Deferred@718282510 after retry RPC@1624474368) -> (continuation of Deferred@687380555 after retry RPC@1422906839) -> (continuation of Deferred@682390013 after retry RPC@1512187157), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6260df4e -> wakeup thread Time-limited test -> (continuation of Deferred@10569739 after retry RPC@1683975678) -> (continuation of Deferred@718282510 after retry RPC@1624474368) -> (continuation of Deferred@687380555 after retry RPC@1422906839) -> (continuation of Deferred@682390013 after retry RPC@1512187157))) after lookup
23:12:55.619 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:12:55.620 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.620 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:55.621 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:12:56.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.555263 22303 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:12:56.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.555792 22303 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:56.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.558229 22303 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:12:56.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:56.562124 22175 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111)
23:12:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:56.566510 22175 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111)
23:12:56.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.571807 22059 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4" is_pre_election: true
23:12:56.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.573602 22177 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b4da2752128f4adba064520c15942e13; no voters: c75eda9306bf472caf0f597e71854313, d65993ec41674e8f976e3ecbb62149e4
23:12:56.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.574359 22303 raft_consensus.cc:2726] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
23:12:56.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.632288 22308 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader c75eda9306bf472caf0f597e71854313)
23:12:56.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.632680 22308 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.634883 22308 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:56.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:56.638707 21991 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111)
23:12:56.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:56.643177 21991 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111)
23:12:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.650480 22243 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "d65993ec41674e8f976e3ecbb62149e4" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "b4da2752128f4adba064520c15942e13" is_pre_election: true
23:12:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.651237 22243 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d65993ec41674e8f976e3ecbb62149e4 in term 4.
23:12:56.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.652518 21991 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4; no voters: c75eda9306bf472caf0f597e71854313
23:12:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.653172 22308 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 5 FOLLOWER]: Leader pre-election won for term 6
23:12:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.653473 22308 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 5 FOLLOWER]: Starting leader election (detected failure of leader c75eda9306bf472caf0f597e71854313)
23:12:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.653770 22308 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 5 FOLLOWER]: Advancing to term 6
23:12:56.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.657928 22308 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:56.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.659377 22308 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 6 election: Requested vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:12:56.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.660876 22243 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "d65993ec41674e8f976e3ecbb62149e4" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "b4da2752128f4adba064520c15942e13"
23:12:56.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.661448 22243 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 4 FOLLOWER]: Advancing to term 6
23:12:56.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:56.662393 21991 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111)
23:12:56.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.668079 22243 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d65993ec41674e8f976e3ecbb62149e4 in term 6.
23:12:56.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.668893 21991 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4; no voters: c75eda9306bf472caf0f597e71854313
23:12:56.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.669816 22308 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 6 FOLLOWER]: Leader election won for term 6
23:12:56.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.671294 22308 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 6 LEADER]: Becoming Leader. State: Replica: d65993ec41674e8f976e3ecbb62149e4, State: Running, Role: LEADER
23:12:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.672245 22308 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:56.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:56.681154 20957 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 reported cstate change: term changed from 5 to 6, leader changed from c75eda9306bf472caf0f597e71854313 (127.20.69.67) to d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65). New cstate: current_term: 6 leader_uuid: "d65993ec41674e8f976e3ecbb62149e4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: UNKNOWN } } }
23:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:57.087334 21991 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 -> Peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Couldn't send request to peer c75eda9306bf472caf0f597e71854313. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:12:57.257 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:57.257 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:57.258 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2046), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [4ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [4ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [26ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [28ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [28ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [45ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [47ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [48ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [48ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [85ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [108ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [111ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [111ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [241ms] refreshing cache from master, [241ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [244ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [247ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [250ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [250ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [401ms] refreshing cache from master, [401ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [404ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [406ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [409ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [409ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [2041ms] refreshing cache from master, [2041ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [2044ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1147778988(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@32415480 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@10569739 after retry RPC@1683975678) -> (continuation of Deferred@718282510 after retry RPC@1624474368) -> (continuation of Deferred@687380555 after retry RPC@1422906839) -> (continuation of Deferred@682390013 after retry RPC@1512187157) -> (continuation of Deferred@1662595974 after retry RPC@676903037), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6260df4e -> wakeup thread Time-limited test -> (continuation of Deferred@10569739 after retry RPC@1683975678) -> (continuation of Deferred@718282510 after retry RPC@1624474368) -> (continuation of Deferred@687380555 after retry RPC@1422906839) -> (continuation of Deferred@682390013 after retry RPC@1512187157) -> (continuation of Deferred@1662595974 after retry RPC@676903037))) after lookup
23:12:57.259 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:57.260 [DEBUG - kudu-nio-0] (Connection.java:757) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] Successfully connected to peer
23:12:57.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:57.272310 22243 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 6 FOLLOWER]: Refusing update from remote peer d65993ec41674e8f976e3ecbb62149e4: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
23:12:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:57.273995 22308 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
23:12:57.285 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb90bdd72, L:/127.0.0.1:50964 - R:/127.20.69.65:45117] using TOKEN/null
23:12:57.350 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.69.67:43603
23:12:57.352 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:12:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:57.354411 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:12:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data
23:12:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/logs
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.67:43603
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.67
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39541
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.67
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:12:57.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:57.687292 22338 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:12:57.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:57.687794 22338 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:12:57.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:57.688221 22338 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:12:57.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:57.720898 22338 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:12:57.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:57.721864 22338 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.67
23:12:57.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:57.757488 22338 tablet_server_runner.cc:78] Tablet server non-default flags:
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.67:43603
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.67
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39541
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.67
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/logs
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:57.758673 22338 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:12:57.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:57.760223 22338 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:12:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:57.779872 22345 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:57.780964 22347 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:57.781613 22338 server_base.cc:1034] running on GCE node
23:12:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:57.779996 22344 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:12:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.055308 22338 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:12:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.057842 22338 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:12:59.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.059264 22338 hybrid_clock.cc:648] HybridClock initialized: now 1783465979059228 us; error 47 us; skew 500 ppm
23:12:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.060139 22338 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:12:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.070487 22338 webserver.cc:460] Webserver started at http://127.20.69.67:39541/ using document root <none> and password file <none>
23:12:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.071533 22338 fs_manager.cc:359] Metadata directory not provided
23:12:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.071815 22338 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:12:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.079831 22338 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.001s	sys 0.004s
23:12:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.084875 22356 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:12:59.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.086046 22338 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
23:12:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.086407 22338 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c75eda9306bf472caf0f597e71854313"
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:30 on dist-test-slave-l596"
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.088910 22338 fs_report.cc:383] FS layout report
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/data
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:12:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:12:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:12:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:12:59.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.135259 22338 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:12:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.136879 22338 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:12:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.137429 22338 kserver.cc:163] Server-wide thread pool size limit: 3276
23:12:59.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.140904 22338 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:12:59.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.147521 22363 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:12:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.155706 22338 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:12:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.155961 22338 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.011s	user 0.003s	sys 0.000s
23:12:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.156315 22338 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:12:59.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.161105 22338 ts_tablet_manager.cc:610] Registered 1 tablets
23:12:59.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.161345 22338 ts_tablet_manager.cc:589] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
23:12:59.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.161767 22363 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Bootstrap starting.
23:12:59.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.233232 22363 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Log is configured to *not* fsync() on all Append() calls
23:12:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:12:59.239871 21991 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 -> Peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Couldn't send request to peer c75eda9306bf472caf0f597e71854313. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:12:59.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.300187 22363 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:12:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.301343 22363 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Bootstrap complete.
23:12:59.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.303552 22363 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Time spent bootstrapping tablet: real 0.142s	user 0.095s	sys 0.044s
23:12:59.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.328727 22363 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:59.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.332249 22363 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: c75eda9306bf472caf0f597e71854313, State: Initialized, Role: FOLLOWER
23:12:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.333292 22363 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:12:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.337900 22363 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Time spent starting tablet: real 0.034s	user 0.034s	sys 0.000s
23:12:59.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.370568 22338 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.67:43603
23:12:59.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.370805 22471 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.67:43603 every 8 connection(s)
23:12:59.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.373256 22338 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:12:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.376678 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 22338
23:12:59.378 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:59.386 [ERROR - Time-limited test] (ResourceLeakDetector.java:327) LEAK: HashedWheelTimer.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:312)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:251)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:224)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:203)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:185)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:296)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:431)
	org.apache.kudu.client.AsyncKuduScanner$AsyncKuduScannerBuilder.build(AsyncKuduScanner.java:1417)
	org.apache.kudu.client.TestMultipleLeaderFailover$1.get(TestMultipleLeaderFailover.java:48)
	org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
	org.apache.kudu.client.TestMultipleLeaderFailover.waitUntilRowCount(TestMultipleLeaderFailover.java:44)
	org.apache.kudu.client.TestMultipleLeaderFailover.testMultipleFailover(TestMultipleLeaderFailover.java:97)
	sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	java.lang.reflect.Method.invoke(Method.java:498)
	org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	java.util.concurrent.FutureTask.run(FutureTask.java:266)
	java.lang.Thread.run(Thread.java:750)
23:12:59.396 [ERROR - Time-limited test] (ResourceLeakDetector.java:327) LEAK: HashedWheelTimer.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:312)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:251)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:224)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:203)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:185)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:296)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:431)
	org.apache.kudu.client.AsyncKuduScanner$AsyncKuduScannerBuilder.build(AsyncKuduScanner.java:1417)
	org.apache.kudu.client.TestMultipleLeaderFailover$1.get(TestMultipleLeaderFailover.java:48)
	org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
	org.apache.kudu.client.TestMultipleLeaderFailover.waitUntilRowCount(TestMultipleLeaderFailover.java:44)
	org.apache.kudu.client.TestMultipleLeaderFailover.testMultipleFailover(TestMultipleLeaderFailover.java:79)
	sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	java.lang.reflect.Method.invoke(Method.java:498)
	org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	java.util.concurrent.FutureTask.run(FutureTask.java:266)
	java.lang.Thread.run(Thread.java:750)
23:12:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.424198 22472 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:12:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.424651 22472 heartbeater.cc:464] Registering TS with master...
23:12:59.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.426138 22472 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:12:59.431 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:12:59.432 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.69.65:45117
23:12:59.433 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:12:59.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.433769 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 21964
23:12:59.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.434231 20957 ts_manager.cc:194] Re-registered known tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:12:59.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.437904 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.67:57905
23:12:59.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.448678 22473 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:12:59.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.449123 22473 heartbeater.cc:464] Registering TS with master...
23:12:59.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.450083 22473 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:12:59.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.454290 20877 ts_manager.cc:194] Re-registered known tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:12:59.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.455343 22474 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:12:59.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.455689 22474 heartbeater.cc:464] Registering TS with master...
23:12:59.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.456209 22474 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:12:59.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:12:59.459713 20802 ts_manager.cc:194] Re-registered known tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:12:59.474 [DEBUG - kudu-nio-0] (Connection.java:239) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] handling channelInactive
23:12:59.475 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state READY due to: connection closed
23:12:59.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:12:59.476 [DEBUG - Time-limited test] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:59.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.479 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.497 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:59.498 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:59.498 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1352937022(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@41c9ea0e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@390fca14 -> wakeup thread Time-limited test)) after lookup
23:12:59.499 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:59.500 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.500 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.500 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.518 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:59.518 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:59.519 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [23ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [24ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [24ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1352937022(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@41c9ea0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@390fca14 -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569))) after lookup
23:12:59.520 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:59.522 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.522 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.522 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.537 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:59.538 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:59.538 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [23ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [24ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [24ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [43ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [46ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [47ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [61ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1352937022(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@41c9ea0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569) -> (continuation of Deferred@1904113293 after retry RPC@994952812), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@390fca14 -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569) -> (continuation of Deferred@1904113293 after retry RPC@994952812))) after lookup
23:12:59.539 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:59.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.540 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.638 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:12:59.638 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:12:59.639 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=162), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [23ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [24ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [24ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [43ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [46ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [47ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [61ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [62ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [64ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [64ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [158ms] refreshing cache from master, [158ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [161ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1352937022(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@41c9ea0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569) -> (continuation of Deferred@1904113293 after retry RPC@994952812) -> (continuation of Deferred@708920464 after retry RPC@728819919), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@390fca14 -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569) -> (continuation of Deferred@1904113293 after retry RPC@994952812) -> (continuation of Deferred@708920464 after retry RPC@728819919))) after lookup
23:12:59.640 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:12:59.641 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.642 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:12:59.642 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:00.017 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:00.017 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:13:00.018 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=541), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [23ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [24ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [24ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [43ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [46ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [47ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [61ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [62ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [64ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [64ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [158ms] refreshing cache from master, [158ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [161ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [163ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [166ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [166ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [537ms] refreshing cache from master, [537ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [540ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1352937022(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@41c9ea0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569) -> (continuation of Deferred@1904113293 after retry RPC@994952812) -> (continuation of Deferred@708920464 after retry RPC@728819919) -> (continuation of Deferred@3631186 after retry RPC@749587979), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@390fca14 -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569) -> (continuation of Deferred@1904113293 after retry RPC@994952812) -> (continuation of Deferred@708920464 after retry RPC@728819919) -> (continuation of Deferred@3631186 after retry RPC@749587979))) after lookup
23:13:00.019 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:13:00.020 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:00.020 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:00.020 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:00.277 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:00.277 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:13:00.278 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=801), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [23ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [24ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [24ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [43ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [46ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [47ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [61ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [62ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [64ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [64ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [158ms] refreshing cache from master, [158ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [161ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [163ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [166ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [166ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [537ms] refreshing cache from master, [537ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [540ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [542ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [544ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [544ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [797ms] refreshing cache from master, [797ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [801ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1352937022(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@41c9ea0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569) -> (continuation of Deferred@1904113293 after retry RPC@994952812) -> (continuation of Deferred@708920464 after retry RPC@728819919) -> (continuation of Deferred@3631186 after retry RPC@749587979) -> (continuation of Deferred@1665773536 after retry RPC@537383530), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@390fca14 -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569) -> (continuation of Deferred@1904113293 after retry RPC@994952812) -> (continuation of Deferred@708920464 after retry RPC@728819919) -> (continuation of Deferred@3631186 after retry RPC@749587979) -> (continuation of Deferred@1665773536 after retry RPC@537383530))) after lookup
23:13:00.279 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:13:00.280 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:00.280 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:00.281 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:00.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.443780 22472 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:13:00.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.490275 22484 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader d65993ec41674e8f976e3ecbb62149e4)
23:13:00.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.490727 22484 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:00.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.492461 22484 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:13:00.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:00.498443 22177 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111)
23:13:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.506830 22426 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313" is_pre_election: true
23:13:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.507652 22426 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4da2752128f4adba064520c15942e13 in term 5.
23:13:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.508884 22175 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, c75eda9306bf472caf0f597e71854313; no voters: d65993ec41674e8f976e3ecbb62149e4
23:13:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.509634 22484 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 6 FOLLOWER]: Leader pre-election won for term 7
23:13:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.510058 22484 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 6 FOLLOWER]: Starting leader election (detected failure of leader d65993ec41674e8f976e3ecbb62149e4)
23:13:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.510426 22484 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 6 FOLLOWER]: Advancing to term 7
23:13:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.515479 22484 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.517127 22484 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 7 election: Requested vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:13:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.517827 22426 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313"
23:13:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.518362 22426 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 5 FOLLOWER]: Advancing to term 7
23:13:00.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:00.521145 22177 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111)
23:13:00.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.525704 22426 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4da2752128f4adba064520c15942e13 in term 7.
23:13:00.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.526640 22175 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, c75eda9306bf472caf0f597e71854313; no voters: d65993ec41674e8f976e3ecbb62149e4
23:13:00.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.527266 22484 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 7 FOLLOWER]: Leader election won for term 7
23:13:00.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.528379 22484 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 7 LEADER]: Becoming Leader. State: Replica: b4da2752128f4adba064520c15942e13, State: Running, Role: LEADER
23:13:00.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.529402 22484 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:00.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:00.537433 20957 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 reported cstate change: term changed from 6 to 7, leader changed from d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65) to b4da2752128f4adba064520c15942e13 (127.20.69.66). New cstate: current_term: 7 leader_uuid: "b4da2752128f4adba064520c15942e13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: HEALTHY } } }
23:13:00.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:00.955914 22177 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 -> Peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Couldn't send request to peer d65993ec41674e8f976e3ecbb62149e4. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:13:01.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:01.006681 22426 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 7 FOLLOWER]: Refusing update from remote peer b4da2752128f4adba064520c15942e13: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
23:13:01.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:01.008090 22484 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
23:13:01.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:01.566172 22177 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:13:03.078 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:03.078 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:03.079 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=3602), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [23ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [24ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [24ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [43ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [46ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [47ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [61ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [62ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [64ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [64ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [158ms] refreshing cache from master, [158ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [161ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [163ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [166ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [166ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [537ms] refreshing cache from master, [537ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [540ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [542ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [544ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [544ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [797ms] refreshing cache from master, [797ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [801ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [802ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [804ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [805ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [3598ms] refreshing cache from master, [3598ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [3602ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1352937022(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@41c9ea0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569) -> (continuation of Deferred@1904113293 after retry RPC@994952812) -> (continuation of Deferred@708920464 after retry RPC@728819919) -> (continuation of Deferred@3631186 after retry RPC@749587979) -> (continuation of Deferred@1665773536 after retry RPC@537383530) -> (continuation of Deferred@911503357 after retry RPC@677281759), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@390fca14 -> wakeup thread Time-limited test -> (continuation of Deferred@1712251939 after retry RPC@1711644569) -> (continuation of Deferred@1904113293 after retry RPC@994952812) -> (continuation of Deferred@708920464 after retry RPC@728819919) -> (continuation of Deferred@3631186 after retry RPC@749587979) -> (continuation of Deferred@1665773536 after retry RPC@537383530) -> (continuation of Deferred@911503357 after retry RPC@677281759))) after lookup
23:13:03.079 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:03.080 [DEBUG - kudu-nio-0] (Connection.java:757) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] Successfully connected to peer
23:13:03.102 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb4304eb3, L:/127.0.0.1:44210 - R:/127.20.69.66:46673] using TOKEN/null
23:13:03.155 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.69.65:45117
23:13:03.156 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:13:03.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:03.157471 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:13:03.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:13:03.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/logs
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:13:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.65:45117
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.65
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45683
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.65
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:13:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:13:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:03.486373 22526 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:13:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:03.486863 22526 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:13:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:03.487444 22526 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:13:03.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:03.519950 22526 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:13:03.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:03.521315 22526 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.65
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:03.561789 22526 tablet_server_runner.cc:78] Tablet server non-default flags:
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.65:45117
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.65
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45683
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:13:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.65
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/logs
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:13:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:13:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:03.563515 22526 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:13:03.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:03.566104 22526 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:13:03.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:03.587054 22538 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:03.704129 22177 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 -> Peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Couldn't send request to peer d65993ec41674e8f976e3ecbb62149e4. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:13:04.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:03.587565 22535 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:04.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:03.587136 22536 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:04.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:04.809532 22537 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:13:04.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.809665 22526 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:13:04.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.813985 22526 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:13:04.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.816725 22526 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:13:04.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.818154 22526 hybrid_clock.cc:648] HybridClock initialized: now 1783465984818084 us; error 96 us; skew 500 ppm
23:13:04.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.819249 22526 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:13:04.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.831638 22526 webserver.cc:460] Webserver started at http://127.20.69.65:45683/ using document root <none> and password file <none>
23:13:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.832557 22526 fs_manager.cc:359] Metadata directory not provided
23:13:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.832749 22526 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:13:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.841341 22526 fs_manager.cc:709] Time spent opening directory manager: real 0.006s	user 0.002s	sys 0.004s
23:13:04.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.846496 22549 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:13:04.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.847777 22526 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.003s
23:13:04.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.848152 22526 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:13:04.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d65993ec41674e8f976e3ecbb62149e4"
23:13:04.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:25 on dist-test-slave-l596"
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.850620 22526 fs_report.cc:383] FS layout report
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/data
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:13:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:13:04.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.906004 22526 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:13:04.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.907608 22526 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:13:04.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.908133 22526 kserver.cc:163] Server-wide thread pool size limit: 3276
23:13:04.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.910791 22526 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:13:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.916505 22556 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:13:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.924712 22526 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:13:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.924964 22526 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
23:13:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.925295 22526 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:13:04.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.930316 22526 ts_tablet_manager.cc:610] Registered 1 tablets
23:13:04.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.930534 22526 ts_tablet_manager.cc:589] Time spent register tablets: real 0.005s	user 0.002s	sys 0.003s
23:13:04.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.930934 22556 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Bootstrap starting.
23:13:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:04.991837 22556 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Log is configured to *not* fsync() on all Append() calls
23:13:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.082787 22556 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:13:05.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.083786 22556 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Bootstrap complete.
23:13:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.085141 22556 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Time spent bootstrapping tablet: real 0.155s	user 0.117s	sys 0.030s
23:13:05.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.110392 22556 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:05.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.114291 22556 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: d65993ec41674e8f976e3ecbb62149e4, State: Initialized, Role: FOLLOWER
23:13:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.115417 22556 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.120985 22556 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Time spent starting tablet: real 0.036s	user 0.033s	sys 0.004s
23:13:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.121181 22526 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.65:45117
23:13:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.121405 22664 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.65:45117 every 8 connection(s)
23:13:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.123764 22526 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:13:05.132 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:13:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.131165 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 22526
23:13:05.192 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:13:05.192 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.69.66:46673
23:13:05.193 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:13:05.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.194288 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 22151
23:13:05.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.194423 22667 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:13:05.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.194886 22667 heartbeater.cc:464] Registering TS with master...
23:13:05.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.196004 22667 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:13:05.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.199679 22619 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 6 FOLLOWER]: Advancing to term 7
23:13:05.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.203027 22665 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:13:05.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.203310 22666 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:13:05.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.203406 22665 heartbeater.cc:464] Registering TS with master...
23:13:05.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.203655 22666 heartbeater.cc:464] Registering TS with master...
23:13:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.204461 22666 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:13:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.204586 22665 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:13:05.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.210197 20802 ts_manager.cc:194] Re-registered known tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:13:05.241 [DEBUG - kudu-nio-0] (Connection.java:239) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] handling channelInactive
23:13:05.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:05.241590 22553 connection.cc:535] server connection from 127.20.69.66:44693 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:13:05.242 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state READY due to: connection closed
23:13:05.242 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:13:05.243 [DEBUG - Time-limited test] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:05.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.246 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.248360 20877 ts_manager.cc:194] Re-registered known tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:13:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.249349 20957 ts_manager.cc:194] Re-registered known tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:13:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.252388 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.65:55043
23:13:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:05.256613 22665 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:13:05.257 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:05.258 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:05.258 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [3ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1712988388(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@4977d3ea -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3468e1ab -> wakeup thread Time-limited test)) after lookup
23:13:05.259 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:05.260 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.260 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.260 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.277 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:05.278 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:05.278 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [3ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [15ms] sending RPC to server b4da2752128f4adba064520c15942e13, [17ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1712988388(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@4977d3ea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3468e1ab -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218))) after lookup
23:13:05.279 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:05.280 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.280 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.280 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.317 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:05.317 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:05.317 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=74), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [3ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [15ms] sending RPC to server b4da2752128f4adba064520c15942e13, [17ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [35ms] sending RPC to server b4da2752128f4adba064520c15942e13, [37ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [73ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1712988388(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@4977d3ea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218) -> (continuation of Deferred@1645674969 after retry RPC@539720902), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3468e1ab -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218) -> (continuation of Deferred@1645674969 after retry RPC@539720902))) after lookup
23:13:05.319 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:05.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.320 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.320 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.377 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:05.378 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:05.378 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=135), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [3ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [15ms] sending RPC to server b4da2752128f4adba064520c15942e13, [17ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [35ms] sending RPC to server b4da2752128f4adba064520c15942e13, [37ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [73ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [75ms] sending RPC to server b4da2752128f4adba064520c15942e13, [77ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [77ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [134ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1712988388(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@4977d3ea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218) -> (continuation of Deferred@1645674969 after retry RPC@539720902) -> (continuation of Deferred@1430155437 after retry RPC@206787020), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3468e1ab -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218) -> (continuation of Deferred@1645674969 after retry RPC@539720902) -> (continuation of Deferred@1430155437 after retry RPC@206787020))) after lookup
23:13:05.379 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:05.381 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.381 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.381 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.497 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:05.498 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:05.498 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=255), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [3ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [15ms] sending RPC to server b4da2752128f4adba064520c15942e13, [17ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [35ms] sending RPC to server b4da2752128f4adba064520c15942e13, [37ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [73ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [75ms] sending RPC to server b4da2752128f4adba064520c15942e13, [77ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [77ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [134ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [135ms] sending RPC to server b4da2752128f4adba064520c15942e13, [138ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [138ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [250ms] refreshing cache from master, [250ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [254ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1712988388(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@4977d3ea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218) -> (continuation of Deferred@1645674969 after retry RPC@539720902) -> (continuation of Deferred@1430155437 after retry RPC@206787020) -> (continuation of Deferred@1709838747 after retry RPC@1931600347), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3468e1ab -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218) -> (continuation of Deferred@1645674969 after retry RPC@539720902) -> (continuation of Deferred@1430155437 after retry RPC@206787020) -> (continuation of Deferred@1709838747 after retry RPC@1931600347))) after lookup
23:13:05.499 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:05.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.500 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.617 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:05.618 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:05.619 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=375), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [3ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [15ms] sending RPC to server b4da2752128f4adba064520c15942e13, [17ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [35ms] sending RPC to server b4da2752128f4adba064520c15942e13, [37ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [73ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [75ms] sending RPC to server b4da2752128f4adba064520c15942e13, [77ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [77ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [134ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [135ms] sending RPC to server b4da2752128f4adba064520c15942e13, [138ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [138ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [250ms] refreshing cache from master, [250ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [254ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [255ms] sending RPC to server b4da2752128f4adba064520c15942e13, [256ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [257ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [371ms] refreshing cache from master, [371ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [374ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1712988388(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@4977d3ea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218) -> (continuation of Deferred@1645674969 after retry RPC@539720902) -> (continuation of Deferred@1430155437 after retry RPC@206787020) -> (continuation of Deferred@1709838747 after retry RPC@1931600347) -> (continuation of Deferred@842963793 after retry RPC@1625834146), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3468e1ab -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218) -> (continuation of Deferred@1645674969 after retry RPC@539720902) -> (continuation of Deferred@1430155437 after retry RPC@206787020) -> (continuation of Deferred@1709838747 after retry RPC@1931600347) -> (continuation of Deferred@842963793 after retry RPC@1625834146))) after lookup
23:13:05.620 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:05.621 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.622 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:05.622 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:06.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.381814 22684 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader b4da2752128f4adba064520c15942e13)
23:13:06.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.382221 22684 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:06.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.384488 22684 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:06.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:06.388644 22358 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:13:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:06.392446 22358 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:13:06.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.398083 22619 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "c75eda9306bf472caf0f597e71854313" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4" is_pre_election: true
23:13:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.399845 22360 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c75eda9306bf472caf0f597e71854313; no voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4
23:13:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.400581 22684 raft_consensus.cc:2726] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
23:13:06.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.713665 22689 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader b4da2752128f4adba064520c15942e13)
23:13:06.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.714135 22689 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:06.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.716193 22689 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:06.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:06.721439 22551 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:13:06.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:06.725695 22551 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:13:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.728960 22426 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "d65993ec41674e8f976e3ecbb62149e4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313" is_pre_election: true
23:13:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.729540 22426 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d65993ec41674e8f976e3ecbb62149e4 in term 7.
23:13:06.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.730517 22551 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c75eda9306bf472caf0f597e71854313, d65993ec41674e8f976e3ecbb62149e4; no voters: b4da2752128f4adba064520c15942e13
23:13:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.731192 22689 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 7 FOLLOWER]: Leader pre-election won for term 8
23:13:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.731422 22689 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 7 FOLLOWER]: Starting leader election (detected failure of leader b4da2752128f4adba064520c15942e13)
23:13:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.731822 22689 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 7 FOLLOWER]: Advancing to term 8
23:13:06.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.736755 22689 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:06.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.738363 22689 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 8 election: Requested vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.739353 22426 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "d65993ec41674e8f976e3ecbb62149e4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313"
23:13:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.739933 22426 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 7 FOLLOWER]: Advancing to term 8
23:13:06.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:06.741813 22551 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:13:06.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.744782 22426 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d65993ec41674e8f976e3ecbb62149e4 in term 8.
23:13:06.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.745698 22551 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c75eda9306bf472caf0f597e71854313, d65993ec41674e8f976e3ecbb62149e4; no voters: b4da2752128f4adba064520c15942e13
23:13:06.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.746389 22689 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 8 FOLLOWER]: Leader election won for term 8
23:13:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.748126 22689 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 8 LEADER]: Becoming Leader. State: Replica: d65993ec41674e8f976e3ecbb62149e4, State: Running, Role: LEADER
23:13:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.749078 22689 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:06.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:06.758484 20957 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 reported cstate change: term changed from 7 to 8, leader changed from b4da2752128f4adba064520c15942e13 (127.20.69.66) to d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65). New cstate: current_term: 8 leader_uuid: "d65993ec41674e8f976e3ecbb62149e4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: UNKNOWN } } }
23:13:07.138 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:07.138 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:13:07.139 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1895), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [3ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [3ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [15ms] sending RPC to server b4da2752128f4adba064520c15942e13, [17ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [35ms] sending RPC to server b4da2752128f4adba064520c15942e13, [37ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [37ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [73ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [75ms] sending RPC to server b4da2752128f4adba064520c15942e13, [77ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [77ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [134ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [135ms] sending RPC to server b4da2752128f4adba064520c15942e13, [138ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [138ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [250ms] refreshing cache from master, [250ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [254ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [255ms] sending RPC to server b4da2752128f4adba064520c15942e13, [256ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [257ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [371ms] refreshing cache from master, [371ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [374ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [376ms] sending RPC to server b4da2752128f4adba064520c15942e13, [379ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [379ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [1894ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1712988388(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@4977d3ea -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218) -> (continuation of Deferred@1645674969 after retry RPC@539720902) -> (continuation of Deferred@1430155437 after retry RPC@206787020) -> (continuation of Deferred@1709838747 after retry RPC@1931600347) -> (continuation of Deferred@842963793 after retry RPC@1625834146) -> (continuation of Deferred@1123330371 after retry RPC@1613438701), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3468e1ab -> wakeup thread Time-limited test -> (continuation of Deferred@697144264 after retry RPC@1644433218) -> (continuation of Deferred@1645674969 after retry RPC@539720902) -> (continuation of Deferred@1430155437 after retry RPC@206787020) -> (continuation of Deferred@1709838747 after retry RPC@1931600347) -> (continuation of Deferred@842963793 after retry RPC@1625834146) -> (continuation of Deferred@1123330371 after retry RPC@1613438701))) after lookup
23:13:07.139 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:13:07.140 [DEBUG - kudu-nio-0] (Connection.java:757) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] Successfully connected to peer
23:13:07.164 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x7dc5dacf, L:/127.0.0.1:38276 - R:/127.20.69.65:45117] using TOKEN/null
23:13:07.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:07.170156 22426 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 8 FOLLOWER]: Refusing update from remote peer d65993ec41674e8f976e3ecbb62149e4: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
23:13:07.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:07.172147 22689 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 28, Time since last communication: 0.000s
23:13:07.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:07.181589 22551 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 -> Peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Couldn't send request to peer b4da2752128f4adba064520c15942e13. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:13:07.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:07.197734 22708 mvcc.cc:205] Tried to move back new op lower bound from 7305076683454328832 to 7305076681732435968. Current Snapshot: MvccSnapshot[applied={T|T < 7305076683454328832}]
23:13:07.253 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.69.66:46673
23:13:07.254 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:07.255888 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/logs
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.66:46673
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.66
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37625
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.66
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:13:07.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:07.575057 22715 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:13:07.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:07.575541 22715 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:13:07.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:07.575973 22715 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:13:07.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:07.612609 22715 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:13:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:07.613723 22715 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.66
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:07.649930 22715 tablet_server_runner.cc:78] Tablet server non-default flags:
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.66:46673
23:13:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.66
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37625
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.66
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/logs
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:13:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:13:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:13:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:13:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:07.651679 22715 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:13:07.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:07.653432 22715 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:13:07.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:07.674437 22722 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:08.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:07.674680 22721 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:08.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:07.679390 22725 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:08.926590 22724 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:13:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.926756 22715 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:13:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.931252 22715 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:13:08.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.933943 22715 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:13:08.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.935359 22715 hybrid_clock.cc:648] HybridClock initialized: now 1783465988935324 us; error 41 us; skew 500 ppm
23:13:08.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.936311 22715 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:13:08.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.943529 22715 webserver.cc:460] Webserver started at http://127.20.69.66:37625/ using document root <none> and password file <none>
23:13:08.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.944578 22715 fs_manager.cc:359] Metadata directory not provided
23:13:08.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.944844 22715 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:13:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.953655 22715 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.003s	sys 0.004s
23:13:08.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.958472 22739 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:13:08.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.959628 22715 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
23:13:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.960039 22715 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:13:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4da2752128f4adba064520c15942e13"
23:13:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:28 on dist-test-slave-l596"
23:13:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:08.963018 22715 fs_report.cc:383] FS layout report
23:13:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:13:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:13:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:13:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/data
23:13:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:13:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:13:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:13:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:13:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:13:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:13:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:13:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:13:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:13:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:13:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:13:09.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.020166 22715 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:13:09.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.022159 22715 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:13:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.022722 22715 kserver.cc:163] Server-wide thread pool size limit: 3276
23:13:09.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.025583 22715 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:13:09.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.031894 22746 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:13:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.040102 22715 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:13:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.040395 22715 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.010s	user 0.000s	sys 0.002s
23:13:09.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.040822 22715 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:13:09.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.046356 22715 ts_tablet_manager.cc:610] Registered 1 tablets
23:13:09.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.046583 22715 ts_tablet_manager.cc:589] Time spent register tablets: real 0.006s	user 0.001s	sys 0.005s
23:13:09.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.046859 22746 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap starting.
23:13:09.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.116320 22746 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Log is configured to *not* fsync() on all Append() calls
23:13:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.233395 22746 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:13:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.234174 22746 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap complete.
23:13:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.235847 22746 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Time spent bootstrapping tablet: real 0.189s	user 0.124s	sys 0.052s
23:13:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.256367 22715 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.66:46673
23:13:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.256619 22853 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.66:46673 every 8 connection(s)
23:13:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.255509 22746 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:09.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.257965 22746 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4da2752128f4adba064520c15942e13, State: Initialized, Role: FOLLOWER
23:13:09.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.259076 22715 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:13:09.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.258841 22746 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:09.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.262594 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 22715
23:13:09.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.262868 22746 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Time spent starting tablet: real 0.027s	user 0.028s	sys 0.000s
23:13:09.263 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:13:09.321 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:13:09.322 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.69.65:45117
23:13:09.322 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:13:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.323348 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 22526
23:13:09.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.345760 22857 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:13:09.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.346262 22857 heartbeater.cc:464] Registering TS with master...
23:13:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.347915 22857 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:13:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.350982 22855 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:13:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.351356 22855 heartbeater.cc:464] Registering TS with master...
23:13:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.352412 22856 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:13:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.352370 22855 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:13:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.352809 22856 heartbeater.cc:464] Registering TS with master...
23:13:09.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.352797 20802 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:09.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.353771 22856 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:13:09.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.355726 20957 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.358928 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.66:50747
23:13:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:09.359134 20877 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:09.370 [DEBUG - kudu-nio-0] (Connection.java:239) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] handling channelInactive
23:13:09.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state READY due to: connection closed
23:13:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:09.371142 22865 negotiation.cc:337] Failed RPC negotiation. Trace:
23:13:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:13:09.304684 (+     0us) reactor.cc:625] Submitting negotiation task for server connection from 127.20.69.65:58063 (local address 127.20.69.66:46673)
23:13:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:13:09.321125 (+ 16441us) server_negotiation.cc:206] Beginning negotiation
23:13:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:13:09.321163 (+    38us) server_negotiation.cc:399] Waiting for connection header
23:13:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:13:09.321244 (+    81us) server_negotiation.cc:407] Connection header received
23:13:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:13:09.323372 (+  2128us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
23:13:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:13:09.323440 (+    68us) server_negotiation.cc:461] Received NEGOTIATE request from client
23:13:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:13:09.325956 (+  2516us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
23:13:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:13:09.326289 (+   333us) server_negotiation.cc:226] Negotiated authn=SASL
23:13:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:13:09.369222 (+ 42933us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.20.69.65:58063: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:13:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":15810,"thread_start_us":20354,"threads_started":1}
23:13:09.373 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:13:09.373 [DEBUG - Time-limited test] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:13:09.375 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.375 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.375 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:09.398 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:13:09.398 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [2ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@153431525(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@3d7bd70f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c74fb68 -> wakeup thread Time-limited test)) after lookup
23:13:09.399 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:13:09.400 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.400 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.400 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.417 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:09.417 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:13:09.418 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [2ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [27ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@153431525(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@3d7bd70f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1573589664 after retry RPC@1491018186), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c74fb68 -> wakeup thread Time-limited test -> (continuation of Deferred@1573589664 after retry RPC@1491018186))) after lookup
23:13:09.419 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:13:09.421 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.421 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.421 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.457 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:09.457 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:13:09.457 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [2ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [27ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [46ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [48ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [48ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@153431525(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@3d7bd70f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1573589664 after retry RPC@1491018186) -> (continuation of Deferred@1186741430 after retry RPC@1741456602), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c74fb68 -> wakeup thread Time-limited test -> (continuation of Deferred@1573589664 after retry RPC@1491018186) -> (continuation of Deferred@1186741430 after retry RPC@1741456602))) after lookup
23:13:09.458 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:13:09.459 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.459 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.557 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:09.557 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:13:09.558 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=184), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [2ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [27ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [46ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [48ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [48ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [85ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [86ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [86ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [180ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [184ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@153431525(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@3d7bd70f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1573589664 after retry RPC@1491018186) -> (continuation of Deferred@1186741430 after retry RPC@1741456602) -> (continuation of Deferred@1193637971 after retry RPC@2057627380), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c74fb68 -> wakeup thread Time-limited test -> (continuation of Deferred@1573589664 after retry RPC@1491018186) -> (continuation of Deferred@1186741430 after retry RPC@1741456602) -> (continuation of Deferred@1193637971 after retry RPC@2057627380))) after lookup
23:13:09.559 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:13:09.560 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.560 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.561 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.897 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:09.897 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:13:09.898 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=524), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [2ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [27ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [46ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [48ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [48ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [85ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [86ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [86ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [180ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [184ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [185ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [187ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [188ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [520ms] refreshing cache from master, [520ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [524ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@153431525(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@3d7bd70f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1573589664 after retry RPC@1491018186) -> (continuation of Deferred@1186741430 after retry RPC@1741456602) -> (continuation of Deferred@1193637971 after retry RPC@2057627380) -> (continuation of Deferred@1478333675 after retry RPC@97733210), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c74fb68 -> wakeup thread Time-limited test -> (continuation of Deferred@1573589664 after retry RPC@1491018186) -> (continuation of Deferred@1186741430 after retry RPC@1741456602) -> (continuation of Deferred@1193637971 after retry RPC@2057627380) -> (continuation of Deferred@1478333675 after retry RPC@97733210))) after lookup
23:13:09.899 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:13:09.900 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.900 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state CONNECTING due to: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:09.900 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117
23:13:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.363252 22855 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:13:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.495105 22869 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader d65993ec41674e8f976e3ecbb62149e4)
23:13:10.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.495692 22869 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:10.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.497359 22869 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:10.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:10.504946 22360 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111)
23:13:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.512459 22809 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "c75eda9306bf472caf0f597e71854313" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "b4da2752128f4adba064520c15942e13" is_pre_election: true
23:13:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.513355 22809 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c75eda9306bf472caf0f597e71854313 in term 7.
23:13:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.514923 22358 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, c75eda9306bf472caf0f597e71854313; no voters: d65993ec41674e8f976e3ecbb62149e4
23:13:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.515538 22869 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 8 FOLLOWER]: Leader pre-election won for term 9
23:13:10.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.515821 22869 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 8 FOLLOWER]: Starting leader election (detected failure of leader d65993ec41674e8f976e3ecbb62149e4)
23:13:10.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.516141 22869 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 8 FOLLOWER]: Advancing to term 9
23:13:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.521201 22869 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.522728 22869 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 9 election: Requested vote from peers d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.523830 22809 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "c75eda9306bf472caf0f597e71854313" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "b4da2752128f4adba064520c15942e13"
23:13:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.524399 22809 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 7 FOLLOWER]: Advancing to term 9
23:13:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:10.526038 22360 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111)
23:13:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.531296 22809 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c75eda9306bf472caf0f597e71854313 in term 9.
23:13:10.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.532147 22358 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, c75eda9306bf472caf0f597e71854313; no voters: d65993ec41674e8f976e3ecbb62149e4
23:13:10.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.532716 22869 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 9 FOLLOWER]: Leader election won for term 9
23:13:10.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.534111 22869 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 9 LEADER]: Becoming Leader. State: Replica: c75eda9306bf472caf0f597e71854313, State: Running, Role: LEADER
23:13:10.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.535111 22869 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:10.545392 20957 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 reported cstate change: term changed from 8 to 9, leader changed from d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65) to c75eda9306bf472caf0f597e71854313 (127.20.69.67). New cstate: current_term: 9 leader_uuid: "c75eda9306bf472caf0f597e71854313" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: UNKNOWN } } }
23:13:11.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:11.007177 22360 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 -> Peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Couldn't send request to peer d65993ec41674e8f976e3ecbb62149e4. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:13:11.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:11.093205 22809 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 9 FOLLOWER]: Refusing update from remote peer c75eda9306bf472caf0f597e71854313: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
23:13:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:11.094626 22869 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
23:13:11.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:11.510353 22360 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:13:11.697 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:11.698 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:11.698 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2325), Traces: [0ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [2ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [2ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [25ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [27ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [27ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [46ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [48ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [48ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [85ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [86ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [86ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [180ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [184ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [185ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [187ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [188ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [520ms] refreshing cache from master, [520ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [524ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [525ms] sending RPC to server d65993ec41674e8f976e3ecbb62149e4, [527ms] received response from server d65993ec41674e8f976e3ecbb62149e4: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [527ms] delaying RPC due to: Network error: Failed to connect to peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117): Connection refused: /127.20.69.65:45117, [2320ms] refreshing cache from master, [2320ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [2324ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@153431525(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@3d7bd70f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1573589664 after retry RPC@1491018186) -> (continuation of Deferred@1186741430 after retry RPC@1741456602) -> (continuation of Deferred@1193637971 after retry RPC@2057627380) -> (continuation of Deferred@1478333675 after retry RPC@97733210) -> (continuation of Deferred@270540349 after retry RPC@1431423925), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c74fb68 -> wakeup thread Time-limited test -> (continuation of Deferred@1573589664 after retry RPC@1491018186) -> (continuation of Deferred@1186741430 after retry RPC@1741456602) -> (continuation of Deferred@1193637971 after retry RPC@2057627380) -> (continuation of Deferred@1478333675 after retry RPC@97733210) -> (continuation of Deferred@270540349 after retry RPC@1431423925))) after lookup
23:13:11.699 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:13:11.701 [DEBUG - kudu-nio-1] (Connection.java:757) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] Successfully connected to peer
23:13:11.722 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x640ce3c1, L:/127.0.0.1:52454 - R:/127.20.69.67:43603] using TOKEN/null
23:13:11.765 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.69.65:45117
23:13:11.765 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:11.766914 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/logs
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:13:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:13:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.65:45117
23:13:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.65
23:13:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:13:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45683
23:13:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.65
23:13:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:13:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:13:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:13:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:13:12.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:12.075361 22903 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:13:12.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:12.075861 22903 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:13:12.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:12.076328 22903 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:13:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:12.108331 22903 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:13:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:12.109163 22903 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.65
23:13:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:12.149556 22903 tablet_server_runner.cc:78] Tablet server non-default flags:
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.65:45117
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.65
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45683
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.65
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/logs
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:13:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:13:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:13:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:13:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:13:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:13:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:13:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:13:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:13:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:13:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:12.150730 22903 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:13:12.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:12.152388 22903 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:13:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:12.173696 22910 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:13.388592 22360 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 -> Peer d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117): Couldn't send request to peer d65993ec41674e8f976e3ecbb62149e4. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.65:45117: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:13:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:12.173842 22912 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:13.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:12.174660 22909 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:13.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:13.404289 22911 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:13:13.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.404409 22903 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:13:13.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.408330 22903 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:13:13.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.411281 22903 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:13:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.412722 22903 hybrid_clock.cc:648] HybridClock initialized: now 1783465993412680 us; error 61 us; skew 500 ppm
23:13:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.413592 22903 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:13:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.420637 22903 webserver.cc:460] Webserver started at http://127.20.69.65:45683/ using document root <none> and password file <none>
23:13:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.421619 22903 fs_manager.cc:359] Metadata directory not provided
23:13:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.421813 22903 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:13:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.430344 22903 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.000s
23:13:13.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.434741 22921 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:13:13.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.435976 22903 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.436270 22903 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d65993ec41674e8f976e3ecbb62149e4"
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:25 on dist-test-slave-l596"
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.439070 22903 fs_report.cc:383] FS layout report
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/wal
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/data
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:13:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:13:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:13:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:13:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:13:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:13:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:13:13.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.485769 22903 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:13:13.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.487254 22903 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:13:13.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.487685 22903 kserver.cc:163] Server-wide thread pool size limit: 3276
23:13:13.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.490247 22903 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:13:13.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.495879 22928 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:13:13.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.503718 22903 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:13:13.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.503952 22903 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
23:13:13.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.504165 22903 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:13:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.509166 22903 ts_tablet_manager.cc:610] Registered 1 tablets
23:13:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.509440 22903 ts_tablet_manager.cc:589] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
23:13:13.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.509832 22928 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Bootstrap starting.
23:13:13.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.592713 22928 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Log is configured to *not* fsync() on all Append() calls
23:13:13.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.700646 22903 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.65:45117
23:13:13.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.700904 23035 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.65:45117 every 8 connection(s)
23:13:13.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.703553 22903 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-0/data/info.pb
23:13:13.707 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:13:13.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.706573 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 22903
23:13:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.722651 22928 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:13:13.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.724068 22928 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Bootstrap complete.
23:13:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.726683 22928 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Time spent bootstrapping tablet: real 0.217s	user 0.162s	sys 0.044s
23:13:13.746 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:13:13.748 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.69.67:43603
23:13:13.748 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:13:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.749182 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 22338
23:13:13.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.788627 23037 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:13:13.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.789162 23037 heartbeater.cc:464] Registering TS with master...
23:13:13.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.790578 23037 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:13:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.794940 23036 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:13:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.799088 23036 heartbeater.cc:464] Registering TS with master...
23:13:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.799993 23036 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:13:13.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.797498 22928 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:13.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.802074 22928 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: d65993ec41674e8f976e3ecbb62149e4, State: Initialized, Role: FOLLOWER
23:13:13.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.803140 22928 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:13.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.807087 23041 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:13:13.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.807510 23041 heartbeater.cc:464] Registering TS with master...
23:13:13.808 [DEBUG - kudu-nio-1] (Connection.java:239) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] handling channelInactive
23:13:13.808 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state READY due to: connection closed
23:13:13.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.808557 23041 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:13:13.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:13:13.813 [DEBUG - Time-limited test] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:13:13.814 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.815 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.815 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.815430 20956 ts_manager.cc:194] Re-registered known tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:13:13.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.815482 20877 ts_manager.cc:194] Re-registered known tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:13:13.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.818925 20956 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.65:33219
23:13:13.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.822770 20802 ts_manager.cc:194] Re-registered known tserver with Master: d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:13:13.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.829996 23036 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:13:13.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:13.830632 22928 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4: Time spent starting tablet: real 0.103s	user 0.029s	sys 0.009s
23:13:13.842 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:13.843 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:13.843 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@794013973(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@e6270ce -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@368f68be -> wakeup thread Time-limited test)) after lookup
23:13:13.844 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:13:13.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.848 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.848 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.857 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:13.857 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:13.858 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [31ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [36ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [36ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [45ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@794013973(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@e6270ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1100070582 after retry RPC@202831010), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@368f68be -> wakeup thread Time-limited test -> (continuation of Deferred@1100070582 after retry RPC@202831010))) after lookup
23:13:13.859 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:13:13.860 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.860 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.860 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.877 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:13.877 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:13.878 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=65), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [31ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [36ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [36ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [45ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [46ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [48ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [48ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [65ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@794013973(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@e6270ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1100070582 after retry RPC@202831010) -> (continuation of Deferred@1259290999 after retry RPC@1374186829), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@368f68be -> wakeup thread Time-limited test -> (continuation of Deferred@1100070582 after retry RPC@202831010) -> (continuation of Deferred@1259290999 after retry RPC@1374186829))) after lookup
23:13:13.878 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:13:13.880 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.880 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.880 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:13.997 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:13.998 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:13.998 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=185), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [31ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [36ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [36ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [45ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [46ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [48ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [48ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [65ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [66ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [68ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [68ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [181ms] refreshing cache from master, [181ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [185ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@794013973(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@e6270ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1100070582 after retry RPC@202831010) -> (continuation of Deferred@1259290999 after retry RPC@1374186829) -> (continuation of Deferred@430614928 after retry RPC@196754925), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@368f68be -> wakeup thread Time-limited test -> (continuation of Deferred@1100070582 after retry RPC@202831010) -> (continuation of Deferred@1259290999 after retry RPC@1374186829) -> (continuation of Deferred@430614928 after retry RPC@196754925))) after lookup
23:13:13.999 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:13:14.000 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:14.000 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:14.000 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:14.258 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:14.258 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)[L],d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:14.259 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=446), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [31ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [36ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [36ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [45ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [46ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [48ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [48ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [65ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [66ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [68ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [68ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [181ms] refreshing cache from master, [181ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [185ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [187ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [188ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [188ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [442ms] refreshing cache from master, [442ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [445ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@794013973(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@e6270ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1100070582 after retry RPC@202831010) -> (continuation of Deferred@1259290999 after retry RPC@1374186829) -> (continuation of Deferred@430614928 after retry RPC@196754925) -> (continuation of Deferred@1764519756 after retry RPC@411944990), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@368f68be -> wakeup thread Time-limited test -> (continuation of Deferred@1100070582 after retry RPC@202831010) -> (continuation of Deferred@1259290999 after retry RPC@1374186829) -> (continuation of Deferred@430614928 after retry RPC@196754925) -> (continuation of Deferred@1764519756 after retry RPC@411944990))) after lookup
23:13:14.260 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] connecting to peer
23:13:14.261 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:14.262 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] cleaning up while in state CONNECTING due to: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:14.262 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c75eda9306bf472caf0f597e71854313(127.20.69.67:43603) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603
23:13:15.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.092749 23050 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader c75eda9306bf472caf0f597e71854313)
23:13:15.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.093159 23050 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:15.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.095289 23050 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:13:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:15.099483 22741 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111)
23:13:15.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:15.103228 22741 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111)
23:13:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.109717 22991 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4" is_pre_election: true
23:13:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.110493 22991 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4da2752128f4adba064520c15942e13 in term 8.
23:13:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.111886 22743 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4; no voters: c75eda9306bf472caf0f597e71854313
23:13:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.112574 23050 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 9 FOLLOWER]: Leader pre-election won for term 10
23:13:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.113013 23050 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 9 FOLLOWER]: Starting leader election (detected failure of leader c75eda9306bf472caf0f597e71854313)
23:13:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.113291 23050 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 9 FOLLOWER]: Advancing to term 10
23:13:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.117668 23050 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.119123 23050 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 10 election: Requested vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117)
23:13:15.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.120313 22991 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "b4da2752128f4adba064520c15942e13" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4"
23:13:15.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.120831 22991 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 8 FOLLOWER]: Advancing to term 10
23:13:15.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:15.122339 22741 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111)
23:13:15.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.127398 22991 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4da2752128f4adba064520c15942e13 in term 10.
23:13:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.128305 22743 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4; no voters: c75eda9306bf472caf0f597e71854313
23:13:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.128973 23050 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 10 FOLLOWER]: Leader election won for term 10
23:13:15.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.130165 23050 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 10 LEADER]: Becoming Leader. State: Replica: b4da2752128f4adba064520c15942e13, State: Running, Role: LEADER
23:13:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.130932 23050 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.140795 20956 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 reported cstate change: term changed from 9 to 10, leader changed from c75eda9306bf472caf0f597e71854313 (127.20.69.67) to b4da2752128f4adba064520c15942e13 (127.20.69.66). New cstate: current_term: 10 leader_uuid: "b4da2752128f4adba064520c15942e13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: HEALTHY } } }
23:13:15.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:15.538365 22741 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 -> Peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Couldn't send request to peer c75eda9306bf472caf0f597e71854313. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:13:15.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.755927 22991 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 10 FOLLOWER]: Refusing update from remote peer b4da2752128f4adba064520c15942e13: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
23:13:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:15.757579 23050 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.001s
23:13:16.178 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:16.178 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:16.178 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2365), Traces: [0ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [3ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [3ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [31ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [36ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [36ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [45ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [46ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [48ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [48ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [65ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [66ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [68ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [68ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [181ms] refreshing cache from master, [181ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [185ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [187ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [188ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [188ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [442ms] refreshing cache from master, [442ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [445ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [447ms] sending RPC to server c75eda9306bf472caf0f597e71854313, [450ms] received response from server c75eda9306bf472caf0f597e71854313: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [450ms] delaying RPC due to: Network error: Failed to connect to peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603): Connection refused: /127.20.69.67:43603, [2362ms] refreshing cache from master, [2362ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [2365ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@794013973(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@e6270ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1100070582 after retry RPC@202831010) -> (continuation of Deferred@1259290999 after retry RPC@1374186829) -> (continuation of Deferred@430614928 after retry RPC@196754925) -> (continuation of Deferred@1764519756 after retry RPC@411944990) -> (continuation of Deferred@809767895 after retry RPC@1191178489), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@368f68be -> wakeup thread Time-limited test -> (continuation of Deferred@1100070582 after retry RPC@202831010) -> (continuation of Deferred@1259290999 after retry RPC@1374186829) -> (continuation of Deferred@430614928 after retry RPC@196754925) -> (continuation of Deferred@1764519756 after retry RPC@411944990) -> (continuation of Deferred@809767895 after retry RPC@1191178489))) after lookup
23:13:16.180 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:16.181 [DEBUG - kudu-nio-0] (Connection.java:757) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] Successfully connected to peer
23:13:16.204 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9fe2d42a, L:/127.0.0.1:57128 - R:/127.20.69.66:46673] using TOKEN/null
23:13:16.244 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.69.67:43603
23:13:16.245 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:13:16.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:16.246592 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/logs
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.67:43603
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.67
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39541
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.67
23:13:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:13:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:13:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:13:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:13:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:16.558516 23079 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:13:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:16.559063 23079 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:13:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:16.559581 23079 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:13:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:16.595386 23079 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:13:16.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:16.596184 23079 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.67
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:16.637908 23079 tablet_server_runner.cc:78] Tablet server non-default flags:
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.67:43603
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.67
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39541
23:13:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.67
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/logs
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:13:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:16.639034 23079 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:13:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:16.640524 23079 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:13:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:16.661044 23086 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:17.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:17.760533 22741 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 -> Peer c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603): Couldn't send request to peer c75eda9306bf472caf0f597e71854313. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.67:43603: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:13:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:17.868862 22850 debug-util.cc:398] Leaking SignalData structure 0x7b08000d0ba0 after lost signal to thread 22716
23:13:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:17.870397 22850 debug-util.cc:398] Leaking SignalData structure 0x7b08000d85e0 after lost signal to thread 22853
23:13:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:16.661814 23087 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:16.663142 23079 server_base.cc:1034] running on GCE node
23:13:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:16.661332 23089 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:17.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.939213 23079 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:13:17.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.942384 23079 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:13:17.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.943903 23079 hybrid_clock.cc:648] HybridClock initialized: now 1783465997943865 us; error 58 us; skew 500 ppm
23:13:17.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.945096 23079 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:13:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.956519 23079 webserver.cc:460] Webserver started at http://127.20.69.67:39541/ using document root <none> and password file <none>
23:13:17.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.957882 23079 fs_manager.cc:359] Metadata directory not provided
23:13:17.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.958184 23079 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:13:17.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.970184 23079 fs_manager.cc:709] Time spent opening directory manager: real 0.007s	user 0.006s	sys 0.000s
23:13:17.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.976370 23098 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:13:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.977756 23079 fs_manager.cc:725] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.004s
23:13:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.978188 23079 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:13:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c75eda9306bf472caf0f597e71854313"
23:13:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:30 on dist-test-slave-l596"
23:13:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:17.981927 23079 fs_report.cc:383] FS layout report
23:13:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:13:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:13:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/wal
23:13:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/data
23:13:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:13:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:13:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:13:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:13:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:13:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:13:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:13:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:13:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:13:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:13:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:13:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.038856 23079 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:13:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.040309 23079 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:13:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.040737 23079 kserver.cc:163] Server-wide thread pool size limit: 3276
23:13:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.044180 23079 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:13:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.051213 23105 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:13:18.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.059973 23079 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:13:18.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.060206 23079 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.003s
23:13:18.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.060421 23079 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:13:18.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.065605 23079 ts_tablet_manager.cc:610] Registered 1 tablets
23:13:18.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.065860 23079 ts_tablet_manager.cc:589] Time spent register tablets: real 0.005s	user 0.001s	sys 0.004s
23:13:18.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.066221 23105 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Bootstrap starting.
23:13:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.158335 23105 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Log is configured to *not* fsync() on all Append() calls
23:13:18.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.299902 23105 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:13:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.301194 23105 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Bootstrap complete.
23:13:18.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.303546 23105 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Time spent bootstrapping tablet: real 0.238s	user 0.165s	sys 0.058s
23:13:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.333300 23105 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.338191 23105 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: c75eda9306bf472caf0f597e71854313, State: Initialized, Role: FOLLOWER
23:13:18.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.339246 23105 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.344427 23105 ts_tablet_manager.cc:1428] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313: Time spent starting tablet: real 0.041s	user 0.032s	sys 0.007s
23:13:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.361279 23079 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.67:43603
23:13:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.361543 23215 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.67:43603 every 8 connection(s)
23:13:18.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.364234 23079 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-2/data/info.pb
23:13:18.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.369403 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 23079
23:13:18.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:13:18.433 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:13:18.436 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.69.66:46673
23:13:18.436 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:13:18.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.437685 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 22715
23:13:18.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.458377 23217 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:13:18.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.458895 23217 heartbeater.cc:464] Registering TS with master...
23:13:18.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.459383 23218 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:13:18.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.459731 23218 heartbeater.cc:464] Registering TS with master...
23:13:18.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.459836 23216 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:13:18.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.460151 23216 heartbeater.cc:464] Registering TS with master...
23:13:18.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.460110 23217 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:13:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.460763 23218 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:13:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.460994 23216 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:13:18.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.464576 20877 ts_manager.cc:194] Re-registered known tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:13:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.464743 20956 ts_manager.cc:194] Re-registered known tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:13:18.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.467330 20802 ts_manager.cc:194] Re-registered known tserver with Master: c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603)
23:13:18.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:18.468734 20956 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.67:47399
23:13:18.480 [DEBUG - kudu-nio-0] (Connection.java:239) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] handling channelInactive
23:13:18.480 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state READY due to: connection closed
23:13:18.480 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:13:18.481 [DEBUG - Time-limited test] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:18.483 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.483 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.483 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.498 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:18.498 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:18.499 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1816552816(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@5aab65f9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f6f9232 -> wakeup thread Time-limited test)) after lookup
23:13:18.500 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:18.501 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.501 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.501 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.517 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:18.517 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:18.518 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [18ms] sending RPC to server b4da2752128f4adba064520c15942e13, [20ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [20ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1816552816(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@5aab65f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@273505073 after retry RPC@1350304208), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f6f9232 -> wakeup thread Time-limited test -> (continuation of Deferred@273505073 after retry RPC@1350304208))) after lookup
23:13:18.519 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:18.520 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.520 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.521 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.557 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:18.558 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:18.558 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [18ms] sending RPC to server b4da2752128f4adba064520c15942e13, [20ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [20ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [38ms] sending RPC to server b4da2752128f4adba064520c15942e13, [40ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [40ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [76ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1816552816(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@5aab65f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@273505073 after retry RPC@1350304208) -> (continuation of Deferred@1725583109 after retry RPC@1444454543), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f6f9232 -> wakeup thread Time-limited test -> (continuation of Deferred@273505073 after retry RPC@1350304208) -> (continuation of Deferred@1725583109 after retry RPC@1444454543))) after lookup
23:13:18.559 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:18.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.560 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.560 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.638 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:18.638 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:18.639 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=158), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [18ms] sending RPC to server b4da2752128f4adba064520c15942e13, [20ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [20ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [38ms] sending RPC to server b4da2752128f4adba064520c15942e13, [40ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [40ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [76ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [78ms] sending RPC to server b4da2752128f4adba064520c15942e13, [79ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [79ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [156ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1816552816(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@5aab65f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@273505073 after retry RPC@1350304208) -> (continuation of Deferred@1725583109 after retry RPC@1444454543) -> (continuation of Deferred@436226867 after retry RPC@1209654847), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f6f9232 -> wakeup thread Time-limited test -> (continuation of Deferred@273505073 after retry RPC@1350304208) -> (continuation of Deferred@1725583109 after retry RPC@1444454543) -> (continuation of Deferred@436226867 after retry RPC@1209654847))) after lookup
23:13:18.640 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:18.641 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.641 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.641 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.878 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:18.878 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673)[L],c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)]}]
23:13:18.878 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=397), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [18ms] sending RPC to server b4da2752128f4adba064520c15942e13, [20ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [20ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [38ms] sending RPC to server b4da2752128f4adba064520c15942e13, [40ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [40ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [76ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [78ms] sending RPC to server b4da2752128f4adba064520c15942e13, [79ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [79ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [156ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [158ms] sending RPC to server b4da2752128f4adba064520c15942e13, [160ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [160ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [393ms] refreshing cache from master, [393ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [396ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1816552816(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@5aab65f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@273505073 after retry RPC@1350304208) -> (continuation of Deferred@1725583109 after retry RPC@1444454543) -> (continuation of Deferred@436226867 after retry RPC@1209654847) -> (continuation of Deferred@1038130348 after retry RPC@990795017), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f6f9232 -> wakeup thread Time-limited test -> (continuation of Deferred@273505073 after retry RPC@1350304208) -> (continuation of Deferred@1725583109 after retry RPC@1444454543) -> (continuation of Deferred@436226867 after retry RPC@1209654847) -> (continuation of Deferred@1038130348 after retry RPC@990795017))) after lookup
23:13:18.879 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] connecting to peer
23:13:18.880 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.880 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:18.880 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4da2752128f4adba064520c15942e13(127.20.69.66:46673) for tablet 43b7940ad4144b84b30a6209edd1fe86: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673
23:13:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.472259 23216 heartbeater.cc:503] Master 127.20.69.124:38915 was elected leader, sending a full tablet report...
23:13:19.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.678217 23228 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:13:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.678653 23228 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:19.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.680799 23228 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65:45117), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:19.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:19.684676 23100 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:13:19.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:19.688661 23100 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:13:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.693641 22991 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "c75eda9306bf472caf0f597e71854313" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "d65993ec41674e8f976e3ecbb62149e4" is_pre_election: true
23:13:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.695055 23102 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c75eda9306bf472caf0f597e71854313; no voters: b4da2752128f4adba064520c15942e13, d65993ec41674e8f976e3ecbb62149e4
23:13:19.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.695807 23228 raft_consensus.cc:2726] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
23:13:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.770653 23233 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader b4da2752128f4adba064520c15942e13)
23:13:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.771080 23233 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:19.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.773310 23233 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:19.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:19.777174 22923 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:13:19.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:19.781780 22923 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:13:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.786168 23168 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "d65993ec41674e8f976e3ecbb62149e4" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313" is_pre_election: true
23:13:19.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.786823 23168 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d65993ec41674e8f976e3ecbb62149e4 in term 9.
23:13:19.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.788002 22923 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c75eda9306bf472caf0f597e71854313, d65993ec41674e8f976e3ecbb62149e4; no voters: b4da2752128f4adba064520c15942e13
23:13:19.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.788697 23233 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 10 FOLLOWER]: Leader pre-election won for term 11
23:13:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.788925 23233 raft_consensus.cc:482] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 10 FOLLOWER]: Starting leader election (detected failure of leader b4da2752128f4adba064520c15942e13)
23:13:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.789139 23233 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 10 FOLLOWER]: Advancing to term 11
23:13:19.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.793273 23233 raft_consensus.cc:504] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.794657 23233 leader_election.cc:288] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 11 election: Requested vote from peers c75eda9306bf472caf0f597e71854313 (127.20.69.67:43603), b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.795524 23168 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43b7940ad4144b84b30a6209edd1fe86" candidate_uuid: "d65993ec41674e8f976e3ecbb62149e4" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "c75eda9306bf472caf0f597e71854313"
23:13:19.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.795996 23168 raft_consensus.cc:3037] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 9 FOLLOWER]: Advancing to term 11
23:13:19.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:19.799192 22923 leader_election.cc:334] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111)
23:13:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.803347 23168 raft_consensus.cc:2454] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d65993ec41674e8f976e3ecbb62149e4 in term 11.
23:13:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.804382 22923 leader_election.cc:302] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c75eda9306bf472caf0f597e71854313, d65993ec41674e8f976e3ecbb62149e4; no voters: b4da2752128f4adba064520c15942e13
23:13:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.805043 23233 raft_consensus.cc:2781] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 11 FOLLOWER]: Leader election won for term 11
23:13:19.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.807145 23233 raft_consensus.cc:686] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [term 11 LEADER]: Becoming Leader. State: Replica: d65993ec41674e8f976e3ecbb62149e4, State: Running, Role: LEADER
23:13:19.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.808203 23233 consensus_queue.cc:237] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.817812 20956 catalog_manager.cc:5495] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 reported cstate change: term changed from 10 to 11, leader changed from b4da2752128f4adba064520c15942e13 (127.20.69.66) to d65993ec41674e8f976e3ecbb62149e4 (127.20.69.65). New cstate: current_term: 11 leader_uuid: "d65993ec41674e8f976e3ecbb62149e4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } health_report { overall_health: UNKNOWN } } }
23:13:19.957 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 43b7940ad4144b84b30a6209edd1fe86 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833' with partition [<start>, <end>)
23:13:19.957 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=43b7940ad4144b84b30a6209edd1fe86@[b4da2752128f4adba064520c15942e13(127.20.69.66:46673),c75eda9306bf472caf0f597e71854313(127.20.69.67:43603),d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)[L]]}]
23:13:19.958 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=43b7940ad4144b84b30a6209edd1fe86, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1477), Traces: [0ms] sending RPC to server b4da2752128f4adba064520c15942e13, [2ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [18ms] sending RPC to server b4da2752128f4adba064520c15942e13, [20ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [20ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [38ms] sending RPC to server b4da2752128f4adba064520c15942e13, [40ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [40ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [76ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [78ms] sending RPC to server b4da2752128f4adba064520c15942e13, [79ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [79ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [153ms] refreshing cache from master, [153ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [156ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [158ms] sending RPC to server b4da2752128f4adba064520c15942e13, [160ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [160ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [393ms] refreshing cache from master, [393ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [396ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, [398ms] sending RPC to server b4da2752128f4adba064520c15942e13, [399ms] received response from server b4da2752128f4adba064520c15942e13: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [399ms] delaying RPC due to: Network error: Failed to connect to peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673): Connection refused: /127.20.69.66:46673, [1472ms] refreshing cache from master, [1472ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.69.124:38915, [1475ms] Sub RPC GetTableLocations: received response from server master-127.20.69.124:38915: OK, deferred=Deferred@1816552816(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/2007160576@5aab65f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@273505073 after retry RPC@1350304208) -> (continuation of Deferred@1725583109 after retry RPC@1444454543) -> (continuation of Deferred@436226867 after retry RPC@1209654847) -> (continuation of Deferred@1038130348 after retry RPC@990795017) -> (continuation of Deferred@466870300 after retry RPC@279399230), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f6f9232 -> wakeup thread Time-limited test -> (continuation of Deferred@273505073 after retry RPC@1350304208) -> (continuation of Deferred@1725583109 after retry RPC@1444454543) -> (continuation of Deferred@436226867 after retry RPC@1209654847) -> (continuation of Deferred@1038130348 after retry RPC@990795017) -> (continuation of Deferred@466870300 after retry RPC@279399230))) after lookup
23:13:19.959 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] connecting to peer
23:13:19.960 [DEBUG - kudu-nio-1] (Connection.java:757) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] Successfully connected to peer
23:13:19.979 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x1165a53e, L:/127.0.0.1:58500 - R:/127.20.69.65:45117] using TOKEN/null
23:13:19.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.988685 23168 raft_consensus.cc:1261] T 43b7940ad4144b84b30a6209edd1fe86 P c75eda9306bf472caf0f597e71854313 [term 11 FOLLOWER]: Refusing update from remote peer d65993ec41674e8f976e3ecbb62149e4: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 42. (index mismatch)
23:13:19.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:19.990515 23238 consensus_queue.cc:1035] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.001s
23:13:19.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:19.991140 22923 consensus_peers.cc:487] T 43b7940ad4144b84b30a6209edd1fe86 P d65993ec41674e8f976e3ecbb62149e4 -> Peer b4da2752128f4adba064520c15942e13 (127.20.69.66:46673): Couldn't send request to peer b4da2752128f4adba064520c15942e13. Status: Network error: Client connection negotiation failed: client connection to 127.20.69.66:46673: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:13:20.060 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.69.66:46673
23:13:20.061 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:20.062765 20757 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/logs
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.66:46673
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.66
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37625
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.66
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:13:20.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:20.389962 23259 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:13:20.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:20.390458 23259 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:13:20.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:20.391049 23259 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:13:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:20.429366 23259 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:13:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:20.430362 23259 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.69.66
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:20.467006 23259 tablet_server_runner.cc:78] Tablet server non-default flags:
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.69.84:35449
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.69.66:46673
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:13:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.69.66
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37625
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.69.126:41549,127.20.69.125:36287,127.20.69.124:38915
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.69.66
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/logs
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:15:51 UTC on ccycloud.quasar-bszuid.root.comops.site
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492329
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:13:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:20.468621 23259 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:13:20.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:20.470414 23259 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:13:20.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:20.491704 23267 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:21.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:20.491716 23268 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:13:20.492487 23270 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:13:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:20.493623 23259 server_base.cc:1034] running on GCE node
23:13:21.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.722714 23259 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:13:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.725363 23259 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:13:21.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.726898 23259 hybrid_clock.cc:648] HybridClock initialized: now 1783466001726850 us; error 69 us; skew 500 ppm
23:13:21.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.727902 23259 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:13:21.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.735396 23259 webserver.cc:460] Webserver started at http://127.20.69.66:37625/ using document root <none> and password file <none>
23:13:21.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.736457 23259 fs_manager.cc:359] Metadata directory not provided
23:13:21.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.736678 23259 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:13:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.745956 23259 fs_manager.cc:709] Time spent opening directory manager: real 0.006s	user 0.007s	sys 0.001s
23:13:21.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.750959 23281 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:13:21.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.752274 23259 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.002s
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.752595 23259 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data,/tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4da2752128f4adba064520c15942e13"
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:12:28 on dist-test-slave-l596"
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.755143 23259 fs_report.cc:383] FS layout report
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/wal
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/data
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:13:21.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.797628 23259 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:13:21.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.799276 23259 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:13:21.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.799695 23259 kserver.cc:163] Server-wide thread pool size limit: 3276
23:13:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.802403 23259 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:13:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.809399 23288 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:13:21.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.817497 23259 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:13:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.817790 23259 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.001s
23:13:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.818116 23259 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:13:21.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.823762 23259 ts_tablet_manager.cc:610] Registered 1 tablets
23:13:21.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.824000 23259 ts_tablet_manager.cc:589] Time spent register tablets: real 0.006s	user 0.001s	sys 0.003s
23:13:21.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.824357 23288 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap starting.
23:13:21.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.921504 23288 log.cc:824] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Log is configured to *not* fsync() on all Append() calls
23:13:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.999696 23259 rpc_server.cc:307] RPC server started. Bound to: 127.20.69.66:46673
23:13:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:21.999878 23395 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.69.66:46673 every 8 connection(s)
23:13:22.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.003618 23259 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklgyMXj/test-tmp/mini-kudu-cluster2894692589017873407/ts-1/data/info.pb
23:13:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.011103 20757 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu as pid 23259
23:13:22.012 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:13:22.066 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:13:22.082 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465950833, tablet=43b7940ad4144b84b30a6209edd1fe86, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:13:22.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.085922 23288 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:13:22.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.087127 23288 tablet_bootstrap.cc:492] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Bootstrap complete.
23:13:22.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.089042 23288 ts_tablet_manager.cc:1397] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13: Time spent bootstrapping tablet: real 0.265s	user 0.204s	sys 0.047s
23:13:22.092 [DEBUG - main] (Connection.java:575) [peer c75eda9306bf472caf0f597e71854313(127.20.69.67:43603)] disconnecting while in state TERMINATED
23:13:22.093 [DEBUG - main] (Connection.java:575) [peer b4da2752128f4adba064520c15942e13(127.20.69.66:46673)] disconnecting while in state TERMINATED
23:13:22.095 [DEBUG - main] (Connection.java:575) [peer master-127.20.69.125:36287(127.20.69.125:36287)] disconnecting while in state READY
23:13:22.099 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.69.125:36287(127.20.69.125:36287)] handling channelInactive
23:13:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.095635 23396 heartbeater.cc:347] Connected to a master server at 127.20.69.124:38915
23:13:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.096134 23396 heartbeater.cc:464] Registering TS with master...
23:13:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.097434 23396 heartbeater.cc:511] Master 127.20.69.124:38915 requested a full tablet report, sending...
23:13:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.097822 23398 heartbeater.cc:347] Connected to a master server at 127.20.69.126:41549
23:13:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.098186 23398 heartbeater.cc:464] Registering TS with master...
23:13:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.099050 23398 heartbeater.cc:511] Master 127.20.69.126:41549 requested a full tablet report, sending...
23:13:22.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.102257 20957 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:22.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.102487 20802 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:22.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.106122 20957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.69.66:47663
23:13:22.107 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.69.125:36287(127.20.69.125:36287)] cleaning up while in state READY due to: connection closed
23:13:22.108 [DEBUG - main] (Connection.java:575) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] disconnecting while in state READY
23:13:22.109 [DEBUG - kudu-nio-1] (Connection.java:239) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] handling channelInactive
23:13:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.110210 23397 heartbeater.cc:347] Connected to a master server at 127.20.69.125:36287
23:13:22.111 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d65993ec41674e8f976e3ecbb62149e4(127.20.69.65:45117)] cleaning up while in state READY due to: connection closed
23:13:22.111 [DEBUG - main] (Connection.java:575) [peer master-127.20.69.124:38915(127.20.69.124:38915)] disconnecting while in state READY
23:13:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.110571 23397 heartbeater.cc:464] Registering TS with master...
23:13:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.111302 23397 heartbeater.cc:511] Master 127.20.69.125:36287 requested a full tablet report, sending...
23:13:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.109557 23288 raft_consensus.cc:348] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.113131 23288 raft_consensus.cc:729] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4da2752128f4adba064520c15942e13, State: Initialized, Role: FOLLOWER
23:13:22.113 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.69.124:38915(127.20.69.124:38915)] handling channelInactive
23:13:22.113 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.69.124:38915(127.20.69.124:38915)] cleaning up while in state READY due to: connection closed
23:13:22.114 [DEBUG - main] (Connection.java:575) [peer master-127.20.69.126:41549(127.20.69.126:41549)] disconnecting while in state READY
23:13:22.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.114883 23288 consensus_queue.cc:260] T 43b7940ad4144b84b30a6209edd1fe86 P b4da2752128f4adba064520c15942e13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c75eda9306bf472caf0f597e71854313" member_type: VOTER last_known_addr { host: "127.20.69.67" port: 43603 } } peers { permanent_uuid: "d65993ec41674e8f976e3ecbb62149e4" member_type: VOTER last_known_addr { host: "127.20.69.65" port: 45117 } } peers { permanent_uuid: "b4da2752128f4adba064520c15942e13" member_type: VOTER last_known_addr { host: "127.20.69.66" port: 46673 } }
23:13:22.117 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.69.126:41549(127.20.69.126:41549)] handling channelInactive
23:13:22.117 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.69.126:41549(127.20.69.126:41549)] cleaning up while in state READY due to: connection closed
23:13:22.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.125094 20877 ts_manager.cc:194] Re-registered known tserver with Master: b4da2752128f4adba064520c15942e13 (127.20.69.66:46673)
23:13:22.126 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:13:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.145226 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 22903
23:13:22.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.149956 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 23259
23:13:22.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.189891 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 23079
23:13:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.229979 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 20769
23:13:22.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.260378 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 20839
23:13:22.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:13:22.290428 20757 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklgyMXj/build/tsan/bin/kudu with pid 20914
23:13:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:13:23Z chronyd exiting

Time: 69.587

OK (1 test)

