JUnit version 4.13.2
.19:58:47.292 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783540727284
19:58:47.355 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
19:58:47.417 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
19:58:47.430 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskvUDI3f/test-tmp
19:58:47.437 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
19:58:47.441 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskvUDI3f/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
19:58:47.621 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

19:58:47.804 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
19:58:47.805 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

19:58:47.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:47.808164  9792 mini_oidc.cc:150] Starting JWKS server
19:58:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:47.810649  9792 webserver.cc:533] Webserver started at http://127.0.0.1:39459/ using document root <none> and password file <none>
19:58:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:47.810925  9792 mini_oidc.cc:186] Starting OIDC Discovery server
19:58:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:47.811892  9792 webserver.cc:533] Webserver started at http://127.0.0.1:35315/ using document root <none> and password file <none>
19:58:47.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T19:58:47Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
19:58:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T19:58:47Z Disabled control of system clock
19:58:47.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:47.836577  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:47.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:47.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/wal
19:58:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/data
19:58:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:58:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:58:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:58:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/logs
19:58:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/data/info.pb
19:58:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:58:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:58:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:58:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:58:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:58:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:58:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:58:47.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:58:47.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.62:44017
19:58:47.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.62
19:58:47.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:47.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:47.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:47.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:47.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:47.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:58:47.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:47.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:58:47.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39459/jwks.json with env {}
19:58:48.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.158507  9803 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:58:48.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.158852  9803 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:58:48.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.158941  9803 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:58:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.167693  9803 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
19:58:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.167798  9803 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:58:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.167860  9803 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
19:58:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.167909  9803 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
19:58:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.167989  9803 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
19:58:48.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.168040  9803 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39459/jwks.json
19:58:48.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.179127  9803 master_runner.cc:387] Master server non-default flags:
19:58:48.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:48.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:48.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:48.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:48.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/data
19:58:48.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/wal
19:58:48.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:58:48.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:48.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:48.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:48.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:58:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.62:44017
19:58:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:58:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:58:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:58:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39459/jwks.json
19:58:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/data/info.pb
19:58:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.62
19:58:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:58:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:58:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:58:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:48.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/logs
19:58:48.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:48.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:58:48.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:58:48.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:58:48.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:58:48.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:58:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:58:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:58:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:58:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:58:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.181041  9803 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:58:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.183347  9803 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:58:48.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.191942  9808 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:48.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.191967  9809 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:48.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.193186  9803 server_base.cc:1061] running on GCE node
19:58:48.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.193641  9811 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:48.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.194414  9803 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:58:48.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.195842  9803 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:58:48.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.197053  9803 hybrid_clock.cc:648] HybridClock initialized: now 1783540728196987 us; error 56 us; skew 500 ppm
19:58:48.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.197365  9803 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:58:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.199430  9803 webserver.cc:533] Webserver started at http://127.9.144.62:42149/ using document root <none> and password file <none>
19:58:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.200083  9803 fs_manager.cc:362] Metadata directory not provided
19:58:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.200171  9803 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:58:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.200713  9803 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
19:58:48.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.203099  9803 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/data/instance:
19:58:48.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6457a2605efd46a8abc553d6bc023f3d"
19:58:48.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:48 on dist-test-slave-trb6"
19:58:48.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.203926  9803 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/wal/instance:
19:58:48.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6457a2605efd46a8abc553d6bc023f3d"
19:58:48.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:48 on dist-test-slave-trb6"
19:58:48.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.209501  9803 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.006s	sys 0.002s
19:58:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.212862  9817 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:58:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.214423  9803 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.004s
19:58:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.214568  9803 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/wal
19:58:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6457a2605efd46a8abc553d6bc023f3d"
19:58:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:48 on dist-test-slave-trb6"
19:58:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.214735  9803 fs_report.cc:389] FS layout report
19:58:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:58:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/wal
19:58:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/wal
19:58:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/data/data
19:58:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:58:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:58:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:58:48.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:58:48.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:58:48.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:58:48.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:58:48.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:58:48.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:58:48.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:58:48.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:58:48.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.235441  9803 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:58:48.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.236266  9803 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:58:48.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.236485  9803 kserver.cc:163] Server-wide thread pool size limit: 3276
19:58:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.255139  9803 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.62:44017
19:58:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.255160  9868 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.62:44017 every 8 connection(s)
19:58:48.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.256886  9803 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/data/info.pb
19:58:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.261834  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 9803
19:58:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.262120  9792 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-0/wal/instance
19:58:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.262243  9869 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:58:48.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.264520  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:48.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:48.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/wal
19:58:48.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/data
19:58:48.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:58:48.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:58:48.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:58:48.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:48.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:48.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:48.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/logs
19:58:48.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/data/info.pb
19:58:48.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:58:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:58:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:58:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:58:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:58:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:58:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:58:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:58:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.61:40553
19:58:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.61
19:58:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:58:48.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:48.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:58:48.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39459/jwks.json with env {}
19:58:48.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.269162  9869 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } has no permanent_uuid. Determining permanent_uuid...
19:58:48.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.286285  9869 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } has no permanent_uuid. Determining permanent_uuid...
19:58:48.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.287523  9819 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.61:40553: connect: Connection refused (error 111)
19:58:48.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.289201  9869 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.61:40553: Network error: Client connection negotiation failed: client connection to 127.9.144.61:40553: connect: Connection refused (error 111)
19:58:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.338490  9869 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } attempt: 1
19:58:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.340305  9869 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.61:40553: Network error: Client connection negotiation failed: client connection to 127.9.144.61:40553: connect: Connection refused (error 111)
19:58:48.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.408589  9869 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } attempt: 2
19:58:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.410794  9869 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.61:40553: Network error: Client connection negotiation failed: client connection to 127.9.144.61:40553: connect: Connection refused (error 111)
19:58:48.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.502005  9869 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } attempt: 3
19:58:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.504217  9869 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.61:40553: Network error: Client connection negotiation failed: client connection to 127.9.144.61:40553: connect: Connection refused (error 111)
19:58:48.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.580313  9871 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:58:48.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.580634  9871 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:58:48.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.580747  9871 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:58:48.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.589221  9871 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
19:58:48.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.589311  9871 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:58:48.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.589383  9871 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
19:58:48.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.589437  9871 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
19:58:48.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.589506  9871 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
19:58:48.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.589550  9871 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39459/jwks.json
19:58:48.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.599920  9871 master_runner.cc:387] Master server non-default flags:
19:58:48.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:48.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:48.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:48.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:48.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/data
19:58:48.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/wal
19:58:48.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:58:48.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:48.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:48.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:48.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:58:48.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.61:40553
19:58:48.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:58:48.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:58:48.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:58:48.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39459/jwks.json
19:58:48.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:48.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:48.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/data/info.pb
19:58:48.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.61
19:58:48.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:58:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:58:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:58:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/logs
19:58:48.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:48.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:58:48.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:58:48.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:58:48.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:58:48.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:58:48.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:58:48.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:58:48.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:58:48.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:58:48.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.602115  9871 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:58:48.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.604549  9871 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:58:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.613372  9880 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.613998  9882 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.613327  9879 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.614287  9871 server_base.cc:1061] running on GCE node
19:58:48.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.615227  9871 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:58:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.616714  9871 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:58:48.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.617918  9871 hybrid_clock.cc:648] HybridClock initialized: now 1783540728617850 us; error 58 us; skew 500 ppm
19:58:48.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.618306  9871 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:58:48.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.620364  9871 webserver.cc:533] Webserver started at http://127.9.144.61:40767/ using document root <none> and password file <none>
19:58:48.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.621258  9871 fs_manager.cc:362] Metadata directory not provided
19:58:48.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.621376  9871 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:58:48.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.621826  9871 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
19:58:48.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.624084  9871 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/data/instance:
19:58:48.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3606b053db3045f1b37444cbfc790d40"
19:58:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:48 on dist-test-slave-trb6"
19:58:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.624966  9871 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/wal/instance:
19:58:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3606b053db3045f1b37444cbfc790d40"
19:58:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:48 on dist-test-slave-trb6"
19:58:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.630079  9871 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.001s	sys 0.003s
19:58:48.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.633252  9888 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:58:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.634472  9871 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
19:58:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.634626  9871 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/wal
19:58:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3606b053db3045f1b37444cbfc790d40"
19:58:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:48 on dist-test-slave-trb6"
19:58:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.634847  9871 fs_report.cc:389] FS layout report
19:58:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:58:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/wal
19:58:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/wal
19:58:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/data/data
19:58:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:58:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:58:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:58:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:58:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:58:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:58:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:58:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:58:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:58:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:58:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:58:48.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.647462  9869 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } attempt: 4
19:58:48.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.648463  9871 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:58:48.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.649349  9869 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.61:40553: Network error: Client connection negotiation failed: client connection to 127.9.144.61:40553: connect: Connection refused (error 111)
19:58:48.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.649375  9871 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:58:48.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.649627  9871 kserver.cc:163] Server-wide thread pool size limit: 3276
19:58:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.668136  9871 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.61:40553
19:58:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.668156  9939 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.61:40553 every 8 connection(s)
19:58:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.669835  9871 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/data/info.pb
19:58:48.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.673846  9940 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:58:48.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.679098  9940 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } has no permanent_uuid. Determining permanent_uuid...
19:58:48.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.679837  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 9871
19:58:48.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.680001  9792 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-1/wal/instance
19:58:48.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.681056  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:48.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:48.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/wal
19:58:48.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/data
19:58:48.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:58:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:58:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:58:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/logs
19:58:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/data/info.pb
19:58:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:58:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:58:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:58:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:58:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:58:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:58:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:58:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:58:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.60:37879
19:58:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.60
19:58:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:58:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:58:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39459/jwks.json with env {}
19:58:48.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.697160  9940 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } has no permanent_uuid. Determining permanent_uuid...
19:58:48.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.707412  9940 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } has no permanent_uuid. Determining permanent_uuid...
19:58:48.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.708336  9891 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.60:37879: connect: Connection refused (error 111)
19:58:48.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.709843  9940 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.60:37879: Network error: Client connection negotiation failed: client connection to 127.9.144.60:37879: connect: Connection refused (error 111)
19:58:48.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.759145  9940 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } attempt: 1
19:58:48.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.760974  9940 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.60:37879: Network error: Client connection negotiation failed: client connection to 127.9.144.60:37879: connect: Connection refused (error 111)
19:58:48.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.829212  9940 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } attempt: 2
19:58:48.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.831027  9940 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.60:37879: Network error: Client connection negotiation failed: client connection to 127.9.144.60:37879: connect: Connection refused (error 111)
19:58:48.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.922243  9940 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } attempt: 3
19:58:48.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.924057  9940 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.60:37879: Network error: Client connection negotiation failed: client connection to 127.9.144.60:37879: connect: Connection refused (error 111)
19:58:48.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.948580  9869 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } attempt: 5
19:58:48.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:48.954442  9869 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } has no permanent_uuid. Determining permanent_uuid...
19:58:48.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.956120  9869 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.60:37879: Network error: Client connection negotiation failed: client connection to 127.9.144.60:37879: connect: Connection refused (error 111)
19:58:48.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.997579  9943 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:58:48.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.997874  9943 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:58:48.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:48.997951  9943 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:58:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.006040  9943 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
19:58:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.006132  9943 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:58:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.006184  9943 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
19:58:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.006222  9943 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
19:58:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.006273  9943 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
19:58:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.006305  9943 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39459/jwks.json
19:58:49.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.007337  9869 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } attempt: 1
19:58:49.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.009409  9869 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.60:37879: Network error: Client connection negotiation failed: client connection to 127.9.144.60:37879: connect: Connection refused (error 111)
19:58:49.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.017390  9943 master_runner.cc:387] Master server non-default flags:
19:58:49.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:49.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/data
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/wal
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.60:37879
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:58:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39459/jwks.json
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/data/info.pb
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.60
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/logs
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:58:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:58:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:58:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:58:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:58:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:58:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:58:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:58:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:58:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.019387  9943 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:58:49.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.021792  9943 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:58:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.030059  9951 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.029919  9950 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.030460  9953 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:49.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.031026  9943 server_base.cc:1061] running on GCE node
19:58:49.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.031860  9943 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:58:49.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.033372  9943 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:58:49.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.034584  9943 hybrid_clock.cc:648] HybridClock initialized: now 1783540729034537 us; error 36 us; skew 500 ppm
19:58:49.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.034971  9943 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:58:49.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.037036  9943 webserver.cc:533] Webserver started at http://127.9.144.60:42115/ using document root <none> and password file <none>
19:58:49.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.037788  9943 fs_manager.cc:362] Metadata directory not provided
19:58:49.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.037892  9943 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:58:49.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.038303  9943 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
19:58:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.040658  9943 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/data/instance:
19:58:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d719ee5c216047059e90a3af743a1958"
19:58:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:49 on dist-test-slave-trb6"
19:58:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.041425  9943 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/wal/instance:
19:58:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d719ee5c216047059e90a3af743a1958"
19:58:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:49 on dist-test-slave-trb6"
19:58:49.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.046198  9943 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.003s	sys 0.001s
19:58:49.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.049217  9959 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:58:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.050462  9943 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
19:58:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.050617  9943 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/wal
19:58:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d719ee5c216047059e90a3af743a1958"
19:58:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:49 on dist-test-slave-trb6"
19:58:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.050854  9943 fs_report.cc:389] FS layout report
19:58:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:58:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/wal
19:58:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/wal
19:58:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/data/data
19:58:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:58:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:58:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:58:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:58:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:58:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:58:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:58:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:58:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:58:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:58:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:58:49.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.067309  9940 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } attempt: 4
19:58:49.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.069273  9940 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.60:37879: Network error: Client connection negotiation failed: client connection to 127.9.144.60:37879: connect: Connection refused (error 111)
19:58:49.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.072189  9943 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:58:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.073385  9943 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:58:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.073690  9943 kserver.cc:163] Server-wide thread pool size limit: 3276
19:58:49.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.077607  9869 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } attempt: 2
19:58:49.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.079500  9869 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.144.60:37879: Network error: Client connection negotiation failed: client connection to 127.9.144.60:37879: connect: Connection refused (error 111)
19:58:49.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.091526  9943 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.60:37879
19:58:49.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.091542 10010 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.60:37879 every 8 connection(s)
19:58:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.093082  9943 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/data/info.pb
19:58:49.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.097039 10011 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:58:49.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.098037  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 9943
19:58:49.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.098220  9792 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/master-2/wal/instance
19:58:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.104059 10011 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } has no permanent_uuid. Determining permanent_uuid...
19:58:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.116012 10011 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } has no permanent_uuid. Determining permanent_uuid...
19:58:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.123030 10011 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } has no permanent_uuid. Determining permanent_uuid...
19:58:49.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.133419 10011 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958: Bootstrap starting.
19:58:49.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.136437 10011 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958: Neither blocks nor log segments found. Creating new log.
19:58:49.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.137837 10011 log.cc:826] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958: Log is configured to *not* fsync() on all Append() calls
19:58:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.140198 10011 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958: No bootstrap required, opened a new log
19:58:49.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.145555 10011 raft_consensus.cc:359] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } }
19:58:49.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.145812 10011 raft_consensus.cc:385] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:58:49.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.145898 10011 raft_consensus.cc:740] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d719ee5c216047059e90a3af743a1958, State: Initialized, Role: FOLLOWER
19:58:49.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.146498 10011 consensus_queue.cc:260] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } }
19:58:49.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.147274 10020 sys_catalog.cc:455] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } } }
19:58:49.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.147485 10020 sys_catalog.cc:458] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [sys.catalog]: This master's current role is: FOLLOWER
19:58:49.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.148510 10011 sys_catalog.cc:565] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [sys.catalog]: configured and running, proceeding with master startup.
19:58:49.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.156749 10031 catalog_manager.cc:1594] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:58:49.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.156860 10031 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:58:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.154675  9943 master_runner.cc:187] Error getting master registration for 127.9.144.62:44017: OK, instance_id { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" instance_seqno: 1783540728250138 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:58:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.160900  9943 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
19:58:49.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.185703  9869 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } attempt: 3
19:58:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.195524  9869 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d: Bootstrap starting.
19:58:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.198500  9869 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d: Neither blocks nor log segments found. Creating new log.
19:58:49.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.199493  9869 log.cc:826] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d: Log is configured to *not* fsync() on all Append() calls
19:58:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.202823  9869 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d: No bootstrap required, opened a new log
19:58:49.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.207716  9869 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } }
19:58:49.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.207965  9869 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:58:49.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.208046  9869 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6457a2605efd46a8abc553d6bc023f3d, State: Initialized, Role: FOLLOWER
19:58:49.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.208606  9869 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } }
19:58:49.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.209486 10034 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } } }
19:58:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.209758 10034 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [sys.catalog]: This master's current role is: FOLLOWER
19:58:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.210510  9869 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [sys.catalog]: configured and running, proceeding with master startup.
19:58:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.221508 10045 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:58:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.222012 10045 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:58:49.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.219564  9803 master_runner.cc:187] Error getting master registration for 127.9.144.61:40553: OK, instance_id { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" instance_seqno: 1783540728663500 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:58:49.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.226003  9803 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
19:58:49.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.239068  9823 tablet.cc:2406] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:58:49.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.326087  9965 tablet.cc:2406] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:58:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.360616 10034 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:58:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.360911 10034 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } }
19:58:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.362839 10034 leader_election.cc:290] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3606b053db3045f1b37444cbfc790d40 (127.9.144.61:40553), d719ee5c216047059e90a3af743a1958 (127.9.144.60:37879)
19:58:49.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.364104  9986 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6457a2605efd46a8abc553d6bc023f3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d719ee5c216047059e90a3af743a1958" is_pre_election: true
19:58:49.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.364656  9986 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6457a2605efd46a8abc553d6bc023f3d in term 0.
19:58:49.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.365281  9820 leader_election.cc:304] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6457a2605efd46a8abc553d6bc023f3d, d719ee5c216047059e90a3af743a1958; no voters: 
19:58:49.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.365705 10034 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 0 FOLLOWER]: Leader pre-election won for term 1
19:58:49.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.365847 10034 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:58:49.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.365954 10034 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 0 FOLLOWER]: Advancing to term 1
19:58:49.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.368502  9940 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } attempt: 5
19:58:49.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.373299 10034 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } }
19:58:49.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.375903 10034 leader_election.cc:290] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [CANDIDATE]: Term 1 election: Requested vote from peers 3606b053db3045f1b37444cbfc790d40 (127.9.144.61:40553), d719ee5c216047059e90a3af743a1958 (127.9.144.60:37879)
19:58:49.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.376399  9986 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6457a2605efd46a8abc553d6bc023f3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d719ee5c216047059e90a3af743a1958"
19:58:49.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.376575  9986 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [term 0 FOLLOWER]: Advancing to term 1
19:58:49.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.379719  9986 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6457a2605efd46a8abc553d6bc023f3d in term 1.
19:58:49.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.381175  9820 leader_election.cc:304] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6457a2605efd46a8abc553d6bc023f3d, d719ee5c216047059e90a3af743a1958; no voters: 
19:58:49.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.390993  9940 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40: Bootstrap starting.
19:58:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.395328 10034 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 1 FOLLOWER]: Leader election won for term 1
19:58:49.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.396458  9940 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40: Neither blocks nor log segments found. Creating new log.
19:58:49.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.398106  9940 log.cc:826] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40: Log is configured to *not* fsync() on all Append() calls
19:58:49.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.403456 10034 raft_consensus.cc:697] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [term 1 LEADER]: Becoming Leader. State: Replica: 6457a2605efd46a8abc553d6bc023f3d, State: Running, Role: LEADER
19:58:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.404398 10034 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } }
19:58:49.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.406589  9940 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40: No bootstrap required, opened a new log
19:58:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.414650 10047 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6457a2605efd46a8abc553d6bc023f3d. Latest consensus state: current_term: 1 leader_uuid: "6457a2605efd46a8abc553d6bc023f3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } } }
19:58:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.415320 10047 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [sys.catalog]: This master's current role is: LEADER
19:58:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.421242  9940 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } }
19:58:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.422748  9940 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:58:49.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.423291  9940 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3606b053db3045f1b37444cbfc790d40, State: Initialized, Role: FOLLOWER
19:58:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.425624 10050 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
19:58:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.426555  9940 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } }
19:58:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.428308 10050 catalog_manager.cc:1520] Initializing Kudu cluster ID...
19:58:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.433059 10051 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } } }
19:58:49.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.435911 10051 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [sys.catalog]: This master's current role is: FOLLOWER
19:58:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.437551  9914 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6457a2605efd46a8abc553d6bc023f3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3606b053db3045f1b37444cbfc790d40"
19:58:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.439220  9914 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [term 0 FOLLOWER]: Advancing to term 1
19:58:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.442564  9914 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6457a2605efd46a8abc553d6bc023f3d in term 1.
19:58:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.437827  9915 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6457a2605efd46a8abc553d6bc023f3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3606b053db3045f1b37444cbfc790d40" is_pre_election: true
19:58:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.442970  9915 raft_consensus.cc:2376] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 6457a2605efd46a8abc553d6bc023f3d in term 1. Re-sending same reply.
19:58:49.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.438014  9940 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [sys.catalog]: configured and running, proceeding with master startup.
19:58:49.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.450429  9986 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [term 1 FOLLOWER]: Refusing update from remote peer 6457a2605efd46a8abc553d6bc023f3d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:58:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.451789 10047 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:58:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.459684  9915 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [term 1 FOLLOWER]: Refusing update from remote peer 6457a2605efd46a8abc553d6bc023f3d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:58:49.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.460942 10047 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:58:49.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.483006 10020 sys_catalog.cc:455] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6457a2605efd46a8abc553d6bc023f3d. Latest consensus state: current_term: 1 leader_uuid: "6457a2605efd46a8abc553d6bc023f3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } } }
19:58:49.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.483271 10020 sys_catalog.cc:458] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [sys.catalog]: This master's current role is: FOLLOWER
19:58:49.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.492786 10052 mvcc.cc:204] Tried to move back new op lower bound from 7305382827816697856 to 7305382827652169728. Current Snapshot: MvccSnapshot[applied={T|T < 7305382827816697856}]
19:58:49.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.497383 10047 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6457a2605efd46a8abc553d6bc023f3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } } }
19:58:49.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.497632 10047 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [sys.catalog]: This master's current role is: LEADER
19:58:49.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.500470 10050 catalog_manager.cc:1383] Generated new cluster ID: fad7878d003b4a91bcfa55c88ac1a5c9
19:58:49.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.500571 10050 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
19:58:49.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.501821 10051 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6457a2605efd46a8abc553d6bc023f3d. Latest consensus state: current_term: 1 leader_uuid: "6457a2605efd46a8abc553d6bc023f3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } } }
19:58:49.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.502038 10051 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [sys.catalog]: This master's current role is: FOLLOWER
19:58:49.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.503468 10047 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6457a2605efd46a8abc553d6bc023f3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } } }
19:58:49.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.503650 10047 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d [sys.catalog]: This master's current role is: LEADER
19:58:49.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.505756 10051 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6457a2605efd46a8abc553d6bc023f3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } } }
19:58:49.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.505955 10051 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40 [sys.catalog]: This master's current role is: FOLLOWER
19:58:49.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.517380 10020 sys_catalog.cc:455] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6457a2605efd46a8abc553d6bc023f3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6457a2605efd46a8abc553d6bc023f3d" member_type: VOTER last_known_addr { host: "127.9.144.62" port: 44017 } } peers { permanent_uuid: "3606b053db3045f1b37444cbfc790d40" member_type: VOTER last_known_addr { host: "127.9.144.61" port: 40553 } } peers { permanent_uuid: "d719ee5c216047059e90a3af743a1958" member_type: VOTER last_known_addr { host: "127.9.144.60" port: 37879 } } }
19:58:49.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.517560 10020 sys_catalog.cc:458] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958 [sys.catalog]: This master's current role is: FOLLOWER
19:58:49.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.519186 10050 catalog_manager.cc:1406] Generated new certificate authority record
19:58:49.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.520814 10050 catalog_manager.cc:1540] Loading token signing keys...
19:58:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.541013 10056 mvcc.cc:204] Tried to move back new op lower bound from 7305382827816697856 to 7305382827652169728. Current Snapshot: MvccSnapshot[applied={T|T < 7305382827816697856}]
19:58:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.556074 10050 catalog_manager.cc:6070] T 00000000000000000000000000000000 P 6457a2605efd46a8abc553d6bc023f3d: Generated new TSK 0
19:58:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.557201 10050 catalog_manager.cc:1550] Initializing in-progress tserver states...
19:58:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.563205 10070 catalog_manager.cc:1295] Loaded cluster ID: fad7878d003b4a91bcfa55c88ac1a5c9
19:58:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.563649 10070 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40: loading cluster ID for follower catalog manager: success
19:58:49.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.567823 10070 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40: acquiring CA information for follower catalog manager: success
19:58:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.570603 10070 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 3606b053db3045f1b37444cbfc790d40: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:58:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.582731  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:58:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/logs
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:58:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.1:0
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.1
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.1
19:58:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:49.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:49.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:49.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:49.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:49.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:58:49.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.942986 10072 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:58:49.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.943281 10072 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:58:49.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.943385 10072 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:58:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.951849 10072 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:58:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.952037 10072 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.1
19:58:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.961998 10072 tablet_server_runner.cc:78] Tablet server non-default flags:
19:58:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.1:0
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.1
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.1
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/logs
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:58:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:58:49.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.964227 10072 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:58:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.966570 10072 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:58:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.975960 10078 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:49.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.982671 10077 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:49.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:49.983376 10080 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:49.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.984289 10072 server_base.cc:1061] running on GCE node
19:58:49.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.984941 10072 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:58:49.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.991277 10072 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:58:49.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.992817 10072 hybrid_clock.cc:648] HybridClock initialized: now 1783540729992711 us; error 108 us; skew 500 ppm
19:58:49.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.993218 10072 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:58:49.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.995714 10072 webserver.cc:533] Webserver started at http://127.9.144.1:40347/ using document root <none> and password file <none>
19:58:49.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.996542 10072 fs_manager.cc:362] Metadata directory not provided
19:58:49.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.996641 10072 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:58:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.997087 10072 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
19:58:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:49.999624 10072 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/instance:
19:58:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d2fa818193a44629aea25a754e9ec74"
19:58:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:49 on dist-test-slave-trb6"
19:58:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.000479 10072 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal/instance:
19:58:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d2fa818193a44629aea25a754e9ec74"
19:58:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:49 on dist-test-slave-trb6"
19:58:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.006007 10072 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.007s	sys 0.000s
19:58:50.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.011597 10086 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:58:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.013206 10072 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.006s	sys 0.001s
19:58:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.013363 10072 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:58:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d2fa818193a44629aea25a754e9ec74"
19:58:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:49 on dist-test-slave-trb6"
19:58:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.013578 10072 fs_report.cc:389] FS layout report
19:58:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/data
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:58:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:58:50.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.049947 10072 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:58:50.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.050732 10072 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:58:50.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.050953 10072 kserver.cc:163] Server-wide thread pool size limit: 3276
19:58:50.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.051854 10072 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:58:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.053400 10072 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:58:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.053483 10072 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
19:58:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.053556 10072 ts_tablet_manager.cc:616] Registered 0 tablets
19:58:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.053602 10072 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
19:58:50.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.082489 10072 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.1:34195
19:58:50.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.082532 10198 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.1:34195 every 8 connection(s)
19:58:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.083777 10072 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:58:50.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.094082  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 10072
19:58:50.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.094276  9792 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal/instance
19:58:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.100484  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:58:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data
19:58:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:58:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:58:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:58:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/logs
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.2:0
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.2
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.2
19:58:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:58:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.100520 10201 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:58:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.100862 10201 heartbeater.cc:461] Registering TS with master...
19:58:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.101930 10201 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:58:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.104344  9905 ts_manager.cc:194] Registered new tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:58:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.107486 10199 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:58:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.107614 10199 heartbeater.cc:461] Registering TS with master...
19:58:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.107961 10199 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:58:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.107504 10200 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:58:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.108667 10200 heartbeater.cc:461] Registering TS with master...
19:58:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.109136 10200 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:58:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.109457  9834 ts_manager.cc:194] Registered new tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:58:50.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.111141  9976 ts_manager.cc:194] Registered new tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:58:50.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.111469  9834 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.1:37211
19:58:50.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.160769 10031 catalog_manager.cc:1295] Loaded cluster ID: fad7878d003b4a91bcfa55c88ac1a5c9
19:58:50.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.160862 10031 catalog_manager.cc:1588] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958: loading cluster ID for follower catalog manager: success
19:58:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.163455 10031 catalog_manager.cc:1610] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958: acquiring CA information for follower catalog manager: success
19:58:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.165138 10031 catalog_manager.cc:1638] T 00000000000000000000000000000000 P d719ee5c216047059e90a3af743a1958: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:58:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.412202 10209 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:58:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.412503 10209 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:58:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.412577 10209 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:58:50.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.420594 10209 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:58:50.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.420732 10209 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.2
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.430173 10209 tablet_server_runner.cc:78] Tablet server non-default flags:
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.2:0
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.2
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.2
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/logs
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:58:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:58:50.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:58:50.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.431941 10209 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:58:50.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.434062 10209 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:58:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.442581 10214 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.442559 10215 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.443455 10217 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:50.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.444536 10209 server_base.cc:1061] running on GCE node
19:58:50.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.445081 10209 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:58:50.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.445952 10209 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:58:50.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.447163 10209 hybrid_clock.cc:648] HybridClock initialized: now 1783540730447113 us; error 41 us; skew 500 ppm
19:58:50.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.447508 10209 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:58:50.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.449725 10209 webserver.cc:533] Webserver started at http://127.9.144.2:36739/ using document root <none> and password file <none>
19:58:50.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.450449 10209 fs_manager.cc:362] Metadata directory not provided
19:58:50.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.450548 10209 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:58:50.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.450994 10209 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
19:58:50.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.453341 10209 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/instance:
19:58:50.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d387b01aa6714e8893e35a67daefebb8"
19:58:50.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:58:50.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.454124 10209 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal/instance:
19:58:50.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d387b01aa6714e8893e35a67daefebb8"
19:58:50.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:58:50.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.459247 10209 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.000s
19:58:50.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.462297 10223 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:58:50.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.463490 10209 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
19:58:50.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.463636 10209 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:58:50.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d387b01aa6714e8893e35a67daefebb8"
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.463843 10209 fs_report.cc:389] FS layout report
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/data
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:58:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:58:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.492573 10209 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:58:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.493513 10209 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:58:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.493796 10209 kserver.cc:163] Server-wide thread pool size limit: 3276
19:58:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.494781 10209 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:58:50.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.496387 10209 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:58:50.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.496486 10209 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
19:58:50.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.496583 10209 ts_tablet_manager.cc:616] Registered 0 tablets
19:58:50.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.496658 10209 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
19:58:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.526506 10209 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.2:39377
19:58:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.526567 10335 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.2:39377 every 8 connection(s)
19:58:50.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.527971 10209 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:58:50.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.528784  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 10209
19:58:50.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.528965  9792 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal/instance
19:58:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.534476  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:58:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data
19:58:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:58:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/logs
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:58:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:58:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.3:0
19:58:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.3
19:58:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.3
19:58:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:58:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.550415 10338 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:58:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.550530 10336 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:58:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.550863 10336 heartbeater.cc:461] Registering TS with master...
19:58:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.551220 10337 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:58:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.551343 10337 heartbeater.cc:461] Registering TS with master...
19:58:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.550868 10338 heartbeater.cc:461] Registering TS with master...
19:58:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.552320 10336 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:58:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.552434 10338 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:58:50.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.552340 10337 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:58:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.554158  9905 ts_manager.cc:194] Registered new tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:58:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.554157  9834 ts_manager.cc:194] Registered new tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:58:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.554406  9976 ts_manager.cc:194] Registered new tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:58:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.555203  9834 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.2:33721
19:58:50.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.848198 10342 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:58:50.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.848518 10342 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:58:50.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.848598 10342 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:58:50.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.856848 10342 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:58:50.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.857030 10342 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.3
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.866842 10342 tablet_server_runner.cc:78] Tablet server non-default flags:
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.3:0
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.3
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.3
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/logs
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:58:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:58:50.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:58:50.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:58:50.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:58:50.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:58:50.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:58:50.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:58:50.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.868494 10342 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:58:50.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.870518 10342 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:58:50.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.879217 10349 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.880147 10348 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:50.880614 10351 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:50.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.881853 10342 server_base.cc:1061] running on GCE node
19:58:50.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.882352 10342 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:58:50.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.883193 10342 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:58:50.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.884440 10342 hybrid_clock.cc:648] HybridClock initialized: now 1783540730884389 us; error 36 us; skew 500 ppm
19:58:50.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.884794 10342 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:58:50.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.886783 10342 webserver.cc:533] Webserver started at http://127.9.144.3:37285/ using document root <none> and password file <none>
19:58:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.887511 10342 fs_manager.cc:362] Metadata directory not provided
19:58:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.887612 10342 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:58:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.888005 10342 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
19:58:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.890322 10342 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/instance:
19:58:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "218dc41941a04327b46d9ca11df80df0"
19:58:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:58:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.891098 10342 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal/instance:
19:58:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "218dc41941a04327b46d9ca11df80df0"
19:58:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:58:50.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.895998 10342 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.001s
19:58:50.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.899171 10357 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:58:50.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.900365 10342 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.000s	sys 0.002s
19:58:50.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.900502 10342 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:58:50.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "218dc41941a04327b46d9ca11df80df0"
19:58:50.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:58:50.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.900724 10342 fs_report.cc:389] FS layout report
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/data
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:58:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:58:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.928654 10342 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:58:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.929553 10342 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:58:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.929860 10342 kserver.cc:163] Server-wide thread pool size limit: 3276
19:58:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.930805 10342 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:58:50.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.932426 10342 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:58:50.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.932520 10342 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
19:58:50.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.932628 10342 ts_tablet_manager.cc:616] Registered 0 tablets
19:58:50.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.932718 10342 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
19:58:50.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.961689 10342 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.3:43859
19:58:50.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.961762 10469 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.3:43859 every 8 connection(s)
19:58:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.963068 10342 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:58:50.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.970602  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 10342
19:58:50.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.970773  9792 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal/instance
19:58:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.978140 10472 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:58:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.978524 10472 heartbeater.cc:461] Registering TS with master...
19:58:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.978672 10471 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:58:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.978785 10471 heartbeater.cc:461] Registering TS with master...
19:58:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.979137 10470 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:58:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.979275 10470 heartbeater.cc:461] Registering TS with master...
19:58:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.979465 10471 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:58:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.979720 10470 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:58:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.980938  9834 ts_manager.cc:194] Registered new tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:58:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.981046 10472 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:58:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.981175  9976 ts_manager.cc:194] Registered new tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:58:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.981992  9834 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.3:42929
19:58:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.982034  9905 ts_manager.cc:194] Registered new tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:58:50.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:50.996959  9792 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
19:58:50.999 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
19:58:51.000 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

19:58:51.008 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.9.144.62"
      port: 44017
    }
    bound_http_address {
      host: "127.9.144.62"
      port: 42149
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.9.144.61"
      port: 40553
    }
    bound_http_address {
      host: "127.9.144.61"
      port: 40767
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.9.144.60"
      port: 37879
    }
    bound_http_address {
      host: "127.9.144.60"
      port: 42115
    }
  }
}

19:58:51.011 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

19:58:51.013 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.9.144.1"
      port: 34195
    }
    bound_http_address {
      host: "127.9.144.1"
      port: 40347
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.9.144.2"
      port: 39377
    }
    bound_http_address {
      host: "127.9.144.2"
      port: 36739
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.9.144.3"
      port: 43859
    }
    bound_http_address {
      host: "127.9.144.3"
      port: 37285
    }
  }
}

19:58:51.014 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
19:58:51.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.113921 10199 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:58:51.194 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
19:58:51.239 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
19:58:51.406 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.144.62:44017(127.9.144.62:44017)] connecting to peer
19:58:51.465 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.144.60:37879(127.9.144.60:37879)] connecting to peer
19:58:51.469 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.144.61:40553(127.9.144.61:40553)] connecting to peer
19:58:51.486 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.9.144.61:40553(127.9.144.61:40553)] Successfully connected to peer
19:58:51.487 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.144.62:44017(127.9.144.62:44017)] Successfully connected to peer
19:58:51.493 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.9.144.60:37879(127.9.144.60:37879)] Successfully connected to peer
19:58:51.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.557343 10336 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:58:51.574 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.9.144.60
19:58:51.574 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.9.144.62
19:58:51.575 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.9.144.61
19:58:51.801 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x4b4940f4, L:/127.0.0.1:41162 - R:/127.9.144.61:40553] using SASL/PLAIN
19:58:51.801 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xdc371c89, L:/127.0.0.1:40694 - R:/127.9.144.62:44017] using SASL/PLAIN
19:58:51.801 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x92e004ac, L:/127.0.0.1:44140 - R:/127.9.144.60:37879] using SASL/PLAIN
19:58:51.834 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
19:58:51.835 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=465), Traces: [0ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.144.62:44017, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.144.60:37879, [96ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.144.61:40553, [440ms] Sub RPC ConnectToMaster: received response from server master-127.9.144.61:40553: OK, [440ms] Sub RPC ConnectToMaster: received response from server master-127.9.144.60:37879: OK, [441ms] Sub RPC ConnectToMaster: received response from server master-127.9.144.62:44017: OK, deferred=null) after lookup
19:58:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.879004  9834 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40694:
19:58:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358"
19:58:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
19:58:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
19:58:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
19:58:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
19:58:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
19:58:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
19:58:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:58:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:51.880658  9834 catalog_manager.cc:7077] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
19:58:51.950 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 57514fb32f144393b711c89382a86254
19:58:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.960300 10271 tablet_service.cc:1511] Processing CreateTablet for tablet 07e9b26c7a2f46edb82f05d9afd8005e (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358 [id=57514fb32f144393b711c89382a86254]), partition=RANGE (key) PARTITION UNBOUNDED
19:58:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.961723 10271 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 07e9b26c7a2f46edb82f05d9afd8005e. 1 dirs total, 0 dirs full, 0 dirs failed
19:58:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.978379 10504 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Bootstrap starting.
19:58:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.981560 10405 tablet_service.cc:1511] Processing CreateTablet for tablet 07e9b26c7a2f46edb82f05d9afd8005e (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358 [id=57514fb32f144393b711c89382a86254]), partition=RANGE (key) PARTITION UNBOUNDED
19:58:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.982303 10504 tablet_bootstrap.cc:654] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Neither blocks nor log segments found. Creating new log.
19:58:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.983366 10405 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 07e9b26c7a2f46edb82f05d9afd8005e. 1 dirs total, 0 dirs full, 0 dirs failed
19:58:51.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.984054 10470 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:58:51.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.985109 10504 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Log is configured to *not* fsync() on all Append() calls
19:58:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.989524 10134 tablet_service.cc:1511] Processing CreateTablet for tablet 07e9b26c7a2f46edb82f05d9afd8005e (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358 [id=57514fb32f144393b711c89382a86254]), partition=RANGE (key) PARTITION UNBOUNDED
19:58:51.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:51.991153 10134 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 07e9b26c7a2f46edb82f05d9afd8005e. 1 dirs total, 0 dirs full, 0 dirs failed
19:58:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.010207 10506 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Bootstrap starting.
19:58:52.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.016287 10506 tablet_bootstrap.cc:654] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Neither blocks nor log segments found. Creating new log.
19:58:52.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.018323 10506 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Log is configured to *not* fsync() on all Append() calls
19:58:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.021983 10504 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: No bootstrap required, opened a new log
19:58:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.022300 10504 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Time spent bootstrapping tablet: real 0.044s	user 0.016s	sys 0.023s
19:58:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.026793 10506 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: No bootstrap required, opened a new log
19:58:52.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.028053 10507 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap starting.
19:58:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.028975 10506 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Time spent bootstrapping tablet: real 0.019s	user 0.008s	sys 0.000s
19:58:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.029237 10504 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.029805 10504 raft_consensus.cc:385] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:58:52.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.030059 10504 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d387b01aa6714e8893e35a67daefebb8, State: Initialized, Role: FOLLOWER
19:58:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.030997 10504 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:52.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.032105 10507 tablet_bootstrap.cc:654] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Neither blocks nor log segments found. Creating new log.
19:58:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.036046 10506 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.036649 10506 raft_consensus.cc:385] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:58:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.036908 10506 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2d2fa818193a44629aea25a754e9ec74, State: Initialized, Role: FOLLOWER
19:58:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.037874 10506 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.038393 10507 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Log is configured to *not* fsync() on all Append() calls
19:58:52.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.039917 10504 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Time spent starting tablet: real 0.018s	user 0.013s	sys 0.003s
19:58:52.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.053720 10506 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Time spent starting tablet: real 0.024s	user 0.020s	sys 0.003s
19:58:52.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.058086 10507 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: No bootstrap required, opened a new log
19:58:52.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.058638 10507 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Time spent bootstrapping tablet: real 0.031s	user 0.015s	sys 0.007s
19:58:52.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.066295 10507 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:52.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.066712 10507 raft_consensus.cc:385] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:58:52.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.066804 10507 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 218dc41941a04327b46d9ca11df80df0, State: Initialized, Role: FOLLOWER
19:58:52.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.067622 10507 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.069386 10507 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Time spent starting tablet: real 0.010s	user 0.005s	sys 0.005s
19:58:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:52.088150 10202 tablet.cc:2406] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:58:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.181747 10512 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:58:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.182056 10512 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:52.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.184306 10512 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:58:52.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.193570 10291 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8" is_pre_election: true
19:58:52.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.194175 10291 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 218dc41941a04327b46d9ca11df80df0 in term 0.
19:58:52.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.194336 10154 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74" is_pre_election: true
19:58:52.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.194805 10154 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 218dc41941a04327b46d9ca11df80df0 in term 0.
19:58:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.194944 10361 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, d387b01aa6714e8893e35a67daefebb8; no voters: 
19:58:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.195406 10512 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 0 FOLLOWER]: Leader pre-election won for term 1
19:58:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.195573 10512 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:58:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.195667 10512 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 0 FOLLOWER]: Advancing to term 1
19:58:52.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.198740 10512 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.199397 10512 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 1 election: Requested vote from peers d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:58:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.199805 10291 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8"
19:58:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.199906 10154 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74"
19:58:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.199985 10291 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 0 FOLLOWER]: Advancing to term 1
19:58:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.200069 10154 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 0 FOLLOWER]: Advancing to term 1
19:58:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.202337 10291 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 218dc41941a04327b46d9ca11df80df0 in term 1.
19:58:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.202696 10361 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, d387b01aa6714e8893e35a67daefebb8; no voters: 
19:58:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.203014 10512 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 1 FOLLOWER]: Leader election won for term 1
19:58:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.203092 10154 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 218dc41941a04327b46d9ca11df80df0 in term 1.
19:58:52.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.204236 10512 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 1 LEADER]: Becoming Leader. State: Replica: 218dc41941a04327b46d9ca11df80df0, State: Running, Role: LEADER
19:58:52.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.205165 10512 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.210988  9834 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 reported cstate change: term changed from 0 to 1, leader changed from <none> to 218dc41941a04327b46d9ca11df80df0 (127.9.144.3). New cstate: current_term: 1 leader_uuid: "218dc41941a04327b46d9ca11df80df0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: UNKNOWN } } }
19:58:52.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:52.216616 10473 tablet.cc:2406] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:58:52.237 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 57514fb32f144393b711c89382a86254
19:58:52.253 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:52.259 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299995, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:52.260 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [8ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=null) after lookup
19:58:52.268 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:58:52.273 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] Successfully connected to peer
19:58:52.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:52.285763 10340 tablet.cc:2406] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:58:52.340 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0x93503ac6, L:/127.0.0.1:35268 - R:/127.9.144.3:43859] using TOKEN/null
19:58:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.351421 10154 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 1 FOLLOWER]: Refusing update from remote peer 218dc41941a04327b46d9ca11df80df0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:58:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.351421 10291 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 1 FOLLOWER]: Refusing update from remote peer 218dc41941a04327b46d9ca11df80df0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:58:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.352573 10512 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:58:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.352870 10515 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:58:52.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.371582 10518 mvcc.cc:204] Tried to move back new op lower bound from 7305382839699202048 to 7305382839116685312. Current Snapshot: MvccSnapshot[applied={T|T < 7305382839699202048}]
19:58:52.475 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:58:52.477 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.144.3:43859
19:58:52.478 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

19:58:52.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.479269  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 10342
19:58:52.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.488247 10465 generic_service.cc:196] Checking for leaks (request via RPC)
19:58:52.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:52.582556  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 10342
19:58:52.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:58:52.604 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] handling channelInactive
19:58:52.605 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state READY due to: connection closed
19:58:52.605 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: connection closed
19:58:52.613 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:52.613 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:52.614 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=11), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [3ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [11ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1864169936(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@9f2c6ce -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@67e5e063 -> wakeup thread Time-limited test)) after lookup
19:58:52.614 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:58:52.616 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.616 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.617 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.633 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:52.634 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:52.634 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [3ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [11ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1864169936(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@9f2c6ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@67e5e063 -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898))) after lookup
19:58:52.634 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:58:52.636 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.636 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.636 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.653 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:52.653 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:52.654 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=51), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [3ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [11ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [32ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [34ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [34ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [51ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1864169936(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@9f2c6ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@67e5e063 -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216))) after lookup
19:58:52.654 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:58:52.656 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.656 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.657 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.694 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:52.694 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:52.695 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=92), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [3ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [11ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [32ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [34ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [34ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [51ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [52ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [54ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [55ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [91ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1864169936(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@9f2c6ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216) -> (continuation of Deferred@361204044 after retry RPC@1425352965), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@67e5e063 -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216) -> (continuation of Deferred@361204044 after retry RPC@1425352965))) after lookup
19:58:52.695 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:58:52.697 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.697 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.697 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.774 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:52.774 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:52.775 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=172), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [3ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [11ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [32ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [34ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [34ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [51ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [52ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [54ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [55ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [91ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [93ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [95ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [95ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [172ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1864169936(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@9f2c6ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216) -> (continuation of Deferred@361204044 after retry RPC@1425352965) -> (continuation of Deferred@917265344 after retry RPC@715095175), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@67e5e063 -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216) -> (continuation of Deferred@361204044 after retry RPC@1425352965) -> (continuation of Deferred@917265344 after retry RPC@715095175))) after lookup
19:58:52.776 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:58:52.777 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.777 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:52.777 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:53.534 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:53.534 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:53.534 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=932), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [3ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [11ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [32ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [34ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [34ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [51ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [52ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [54ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [55ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [91ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [93ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [95ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [95ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [172ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [173ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [175ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [175ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [929ms] refreshing cache from master, [929ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [931ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1864169936(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@9f2c6ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216) -> (continuation of Deferred@361204044 after retry RPC@1425352965) -> (continuation of Deferred@917265344 after retry RPC@715095175) -> (continuation of Deferred@576026273 after retry RPC@653937152), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@67e5e063 -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216) -> (continuation of Deferred@361204044 after retry RPC@1425352965) -> (continuation of Deferred@917265344 after retry RPC@715095175) -> (continuation of Deferred@576026273 after retry RPC@653937152))) after lookup
19:58:53.535 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:58:53.536 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:53.536 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:53.536 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:53.714 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:53.714 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:53.715 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1112), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [3ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [11ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [32ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [34ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [34ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [51ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [52ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [54ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [55ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [91ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [93ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [95ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [95ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [172ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [173ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [175ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [175ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [929ms] refreshing cache from master, [929ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [931ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [933ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [934ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [934ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1108ms] refreshing cache from master, [1108ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [1112ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1864169936(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@9f2c6ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216) -> (continuation of Deferred@361204044 after retry RPC@1425352965) -> (continuation of Deferred@917265344 after retry RPC@715095175) -> (continuation of Deferred@576026273 after retry RPC@653937152) -> (continuation of Deferred@791791500 after retry RPC@1542398496), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@67e5e063 -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216) -> (continuation of Deferred@361204044 after retry RPC@1425352965) -> (continuation of Deferred@917265344 after retry RPC@715095175) -> (continuation of Deferred@576026273 after retry RPC@653937152) -> (continuation of Deferred@791791500 after retry RPC@1542398496))) after lookup
19:58:53.716 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:58:53.717 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:53.717 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:53.717 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:58:53.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.923034 10537 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 218dc41941a04327b46d9ca11df80df0)
19:58:53.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.923211 10537 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:53.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.923241 10538 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 218dc41941a04327b46d9ca11df80df0)
19:58:53.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.923445 10538 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:53.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.924655 10537 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:58:53.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.924857 10538 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:58:53.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:53.926146 10088 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:58:53.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:53.944187 10088 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:58:53.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:53.945660 10225 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:58:53.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:53.948072 10225 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:58:53.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.949867 10154 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "d387b01aa6714e8893e35a67daefebb8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74" is_pre_election: true
19:58:53.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.950096 10154 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d387b01aa6714e8893e35a67daefebb8 in term 1.
19:58:53.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.950708 10227 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2d2fa818193a44629aea25a754e9ec74, d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0
19:58:53.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.951174 10538 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 1 FOLLOWER]: Leader pre-election won for term 2
19:58:53.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.951478 10538 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 218dc41941a04327b46d9ca11df80df0)
19:58:53.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.951601 10538 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 1 FOLLOWER]: Advancing to term 2
19:58:53.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.953454 10290 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "2d2fa818193a44629aea25a754e9ec74" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8" is_pre_election: true
19:58:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.954771 10538 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.955188 10290 raft_consensus.cc:2393] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 2d2fa818193a44629aea25a754e9ec74 in current term 2: Already voted for candidate d387b01aa6714e8893e35a67daefebb8 in this term.
19:58:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.955513 10538 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 2 election: Requested vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:58:53.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.956108 10090 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2d2fa818193a44629aea25a754e9ec74; no voters: 218dc41941a04327b46d9ca11df80df0, d387b01aa6714e8893e35a67daefebb8
19:58:53.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.956277 10154 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "d387b01aa6714e8893e35a67daefebb8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74"
19:58:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.956455 10154 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 1 FOLLOWER]: Advancing to term 2
19:58:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:53.957867 10225 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:58:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.958946 10154 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d387b01aa6714e8893e35a67daefebb8 in term 2.
19:58:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.959098 10537 raft_consensus.cc:2749] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
19:58:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.959721 10227 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2d2fa818193a44629aea25a754e9ec74, d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0
19:58:53.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.960031 10538 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 FOLLOWER]: Leader election won for term 2
19:58:53.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.960476 10538 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 LEADER]: Becoming Leader. State: Replica: d387b01aa6714e8893e35a67daefebb8, State: Running, Role: LEADER
19:58:53.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.961031 10538 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:53.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:53.964669  9834 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 reported cstate change: term changed from 1 to 2, leader changed from 218dc41941a04327b46d9ca11df80df0 (127.9.144.3) to d387b01aa6714e8893e35a67daefebb8 (127.9.144.2). New cstate: current_term: 2 leader_uuid: "d387b01aa6714e8893e35a67daefebb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: UNKNOWN } } }
19:58:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:54.343036 10154 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 2 FOLLOWER]: Refusing update from remote peer d387b01aa6714e8893e35a67daefebb8: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
19:58:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:54.343708 10538 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
19:58:54.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:54.347347 10225 consensus_peers.cc:597] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 -> Peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Couldn't send request to peer 218dc41941a04327b46d9ca11df80df0. Status: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:58:55.634 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:55.634 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:58:55.635 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=16, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=3032), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [3ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [11ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [32ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [34ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [34ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [51ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [52ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [54ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [55ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [91ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [93ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [95ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [95ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [172ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [173ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [175ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [175ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [929ms] refreshing cache from master, [929ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [931ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [933ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [934ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [934ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1108ms] refreshing cache from master, [1108ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [1112ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [1113ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [1115ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1115ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [3029ms] refreshing cache from master, [3029ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [3032ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1864169936(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@9f2c6ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216) -> (continuation of Deferred@361204044 after retry RPC@1425352965) -> (continuation of Deferred@917265344 after retry RPC@715095175) -> (continuation of Deferred@576026273 after retry RPC@653937152) -> (continuation of Deferred@791791500 after retry RPC@1542398496) -> (continuation of Deferred@447912158 after retry RPC@946751370), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@67e5e063 -> wakeup thread Time-limited test -> (continuation of Deferred@2105650866 after retry RPC@739029898) -> (continuation of Deferred@1375182848 after retry RPC@999681216) -> (continuation of Deferred@361204044 after retry RPC@1425352965) -> (continuation of Deferred@917265344 after retry RPC@715095175) -> (continuation of Deferred@576026273 after retry RPC@653937152) -> (continuation of Deferred@791791500 after retry RPC@1542398496) -> (continuation of Deferred@447912158 after retry RPC@946751370))) after lookup
19:58:55.636 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:58:55.637 [DEBUG - kudu-nio-3] (Connection.java:757) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] Successfully connected to peer
19:58:55.654 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0x7877617b, L:/127.0.0.1:54800 - R:/127.9.144.2:39377] using TOKEN/null
19:58:55.681 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.144.3:43859
19:58:55.683 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

19:58:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:55.684510  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:58:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data
19:58:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/logs
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.3:43859
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.3
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37285
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.3
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:58:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:55.999068 10571 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:58:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:55.999389 10571 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:58:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:55.999521 10571 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:58:56.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:56.008092 10571 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:58:56.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:56.008291 10571 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.3
19:58:56.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.018505 10571 tablet_server_runner.cc:78] Tablet server non-default flags:
19:58:56.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:56.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.3:43859
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.3
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37285
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.3
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/logs
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:58:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:58:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:58:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:58:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:58:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.020627 10571 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:58:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.022454 10571 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:58:56.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:56.031680 10576 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:56.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:56.031703 10579 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:56.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:56.032459 10577 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:56.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.033427 10571 server_base.cc:1061] running on GCE node
19:58:56.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.034058 10571 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:58:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.034909 10571 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:58:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.036147 10571 hybrid_clock.cc:648] HybridClock initialized: now 1783540736036084 us; error 45 us; skew 500 ppm
19:58:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.036526 10571 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:58:56.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.039043 10571 webserver.cc:533] Webserver started at http://127.9.144.3:37285/ using document root <none> and password file <none>
19:58:56.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.039930 10571 fs_manager.cc:362] Metadata directory not provided
19:58:56.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.040060 10571 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:58:56.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.045469 10571 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.000s	sys 0.003s
19:58:56.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.048120 10585 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:58:56.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.049614 10571 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.049760 10571 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "218dc41941a04327b46d9ca11df80df0"
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.050513 10571 fs_report.cc:389] FS layout report
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/data
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:58:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:58:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:58:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:58:56.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.064450 10571 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:58:56.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.065204 10571 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:58:56.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.065424 10571 kserver.cc:163] Server-wide thread pool size limit: 3276
19:58:56.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.066210 10571 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:58:56.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.068105 10592 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
19:58:56.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.073751 10571 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
19:58:56.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.073829 10571 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
19:58:56.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.073913 10571 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
19:58:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.076671 10571 ts_tablet_manager.cc:616] Registered 1 tablets
19:58:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.076756 10571 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
19:58:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.077241 10592 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap starting.
19:58:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.098898 10592 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Log is configured to *not* fsync() on all Append() calls
19:58:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.116173 10592 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:58:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.116789 10592 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap complete.
19:58:56.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.117995 10592 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Time spent bootstrapping tablet: real 0.041s	user 0.025s	sys 0.015s
19:58:56.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.118229 10571 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.3:43859
19:58:56.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.118388 10699 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.3:43859 every 8 connection(s)
19:58:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.119714 10571 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:58:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.125934 10592 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.126976 10592 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 218dc41941a04327b46d9ca11df80df0, State: Initialized, Role: FOLLOWER
19:58:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.127710 10592 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:56.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:58:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.128707  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 10571
19:58:56.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.135387 10592 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Time spent starting tablet: real 0.017s	user 0.009s	sys 0.007s
19:58:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.139355 10701 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:58:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.139739 10701 heartbeater.cc:461] Registering TS with master...
19:58:56.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.140341 10700 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:58:56.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.140473 10700 heartbeater.cc:461] Registering TS with master...
19:58:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.140841 10701 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:58:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.140965 10700 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:58:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.141139 10702 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:58:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.141255 10702 heartbeater.cc:461] Registering TS with master...
19:58:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.141706 10702 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:58:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.144028  9834 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:58:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.144302  9976 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:58:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.144320  9904 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:58:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.145578  9834 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.3:58173
19:58:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.147966 10700 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:58:56.150 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:58:56.151 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.144.2:39377
19:58:56.151 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

19:58:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.151741  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 10209
19:58:56.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.159515 10331 generic_service.cc:196] Checking for leaks (request via RPC)
19:58:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:56.253007  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 10209
19:58:56.277 [DEBUG - kudu-nio-3] (Connection.java:239) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] handling channelInactive
19:58:56.278 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:58:56.278 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state READY due to: connection closed
19:58:56.279 [DEBUG - Time-limited test] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:58:56.280 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.280 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.280 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.293 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:56.293 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:58:56.294 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [2ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1462915264(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@1d2a1fbb -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@99f8f16 -> wakeup thread Time-limited test)) after lookup
19:58:56.294 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:58:56.295 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.295 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.295 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.314 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:56.314 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:58:56.314 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [2ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [17ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [17ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1462915264(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@1d2a1fbb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@446512903 after retry RPC@1871701117), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@99f8f16 -> wakeup thread Time-limited test -> (continuation of Deferred@446512903 after retry RPC@1871701117))) after lookup
19:58:56.315 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:58:56.316 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.316 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.316 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.333 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:56.333 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:58:56.334 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=55), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [2ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [17ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [17ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [36ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [38ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [38ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1462915264(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@1d2a1fbb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@446512903 after retry RPC@1871701117) -> (continuation of Deferred@1452562332 after retry RPC@1185479999), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@99f8f16 -> wakeup thread Time-limited test -> (continuation of Deferred@446512903 after retry RPC@1871701117) -> (continuation of Deferred@1452562332 after retry RPC@1185479999))) after lookup
19:58:56.335 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:58:56.336 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.336 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.337 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.453 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:56.453 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:58:56.454 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=175), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [2ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [17ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [17ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [36ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [38ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [38ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [57ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [59ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [59ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [172ms] refreshing cache from master, [172ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [175ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1462915264(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@1d2a1fbb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@446512903 after retry RPC@1871701117) -> (continuation of Deferred@1452562332 after retry RPC@1185479999) -> (continuation of Deferred@346628318 after retry RPC@962150028), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@99f8f16 -> wakeup thread Time-limited test -> (continuation of Deferred@446512903 after retry RPC@1871701117) -> (continuation of Deferred@1452562332 after retry RPC@1185479999) -> (continuation of Deferred@346628318 after retry RPC@962150028))) after lookup
19:58:56.455 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:58:56.456 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.456 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.456 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.713 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:56.714 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:58:56.714 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=435), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [2ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [17ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [17ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [36ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [38ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [38ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [57ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [59ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [59ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [172ms] refreshing cache from master, [172ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [175ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [176ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [178ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [178ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [432ms] refreshing cache from master, [432ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [435ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1462915264(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@1d2a1fbb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@446512903 after retry RPC@1871701117) -> (continuation of Deferred@1452562332 after retry RPC@1185479999) -> (continuation of Deferred@346628318 after retry RPC@962150028) -> (continuation of Deferred@1201707748 after retry RPC@878468493), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@99f8f16 -> wakeup thread Time-limited test -> (continuation of Deferred@446512903 after retry RPC@1871701117) -> (continuation of Deferred@1452562332 after retry RPC@1185479999) -> (continuation of Deferred@346628318 after retry RPC@962150028) -> (continuation of Deferred@1201707748 after retry RPC@878468493))) after lookup
19:58:56.715 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:58:56.716 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.716 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:56.716 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:58:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.492254 10714 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:58:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.492512 10714 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:57.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.494050 10714 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:58:57.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:57.495679 10589 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:58:57.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:57.497243 10589 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:58:57.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.500728 10154 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74" is_pre_election: true
19:58:57.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.501454 10589 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0; no voters: 2d2fa818193a44629aea25a754e9ec74, d387b01aa6714e8893e35a67daefebb8
19:58:57.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.501773 10714 raft_consensus.cc:2749] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
19:58:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.641659 10719 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader d387b01aa6714e8893e35a67daefebb8)
19:58:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.641863 10719 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.642633 10719 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:58:57.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:57.644769 10090 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:58:57.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.650238 10654 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "2d2fa818193a44629aea25a754e9ec74" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "218dc41941a04327b46d9ca11df80df0" is_pre_election: true
19:58:57.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.650619 10654 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2d2fa818193a44629aea25a754e9ec74 in term 1.
19:58:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.651440 10088 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, 2d2fa818193a44629aea25a754e9ec74; no voters: d387b01aa6714e8893e35a67daefebb8
19:58:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.651739 10719 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 2 FOLLOWER]: Leader pre-election won for term 3
19:58:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.651867 10719 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 2 FOLLOWER]: Starting leader election (detected failure of leader d387b01aa6714e8893e35a67daefebb8)
19:58:57.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.651968 10719 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 2 FOLLOWER]: Advancing to term 3
19:58:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.654006 10719 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:57.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.654594 10719 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 3 election: Requested vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:58:57.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.655118 10654 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "2d2fa818193a44629aea25a754e9ec74" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "218dc41941a04327b46d9ca11df80df0"
19:58:57.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.655328 10654 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 1 FOLLOWER]: Advancing to term 3
19:58:57.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:57.656064 10090 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:58:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.659166 10654 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2d2fa818193a44629aea25a754e9ec74 in term 3.
19:58:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.659569 10088 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, 2d2fa818193a44629aea25a754e9ec74; no voters: d387b01aa6714e8893e35a67daefebb8
19:58:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.659835 10719 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 3 FOLLOWER]: Leader election won for term 3
19:58:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.660306 10719 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 3 LEADER]: Becoming Leader. State: Replica: 2d2fa818193a44629aea25a754e9ec74, State: Running, Role: LEADER
19:58:57.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.660786 10719 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:57.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:57.664371  9834 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 reported cstate change: term changed from 2 to 3, leader changed from d387b01aa6714e8893e35a67daefebb8 (127.9.144.2) to 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1). New cstate: current_term: 3 leader_uuid: "2d2fa818193a44629aea25a754e9ec74" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: HEALTHY } } }
19:58:58.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:58.142542 10090 consensus_peers.cc:597] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 -> Peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Couldn't send request to peer d387b01aa6714e8893e35a67daefebb8. Status: Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:58:58.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.229640 10654 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 3 FOLLOWER]: Refusing update from remote peer 2d2fa818193a44629aea25a754e9ec74: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
19:58:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.230463 10719 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [LEADER]: Connected to new peer: Peer: permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
19:58:58.474 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:58.475 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:58.475 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=2196), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [2ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [17ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [17ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [36ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [38ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [38ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [57ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [59ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [59ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [172ms] refreshing cache from master, [172ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [175ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [176ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [178ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [178ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [432ms] refreshing cache from master, [432ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [435ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [436ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [438ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [438ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2193ms] refreshing cache from master, [2193ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [2196ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1462915264(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@1d2a1fbb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@446512903 after retry RPC@1871701117) -> (continuation of Deferred@1452562332 after retry RPC@1185479999) -> (continuation of Deferred@346628318 after retry RPC@962150028) -> (continuation of Deferred@1201707748 after retry RPC@878468493) -> (continuation of Deferred@1758687597 after retry RPC@1043720655), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@99f8f16 -> wakeup thread Time-limited test -> (continuation of Deferred@446512903 after retry RPC@1871701117) -> (continuation of Deferred@1452562332 after retry RPC@1185479999) -> (continuation of Deferred@346628318 after retry RPC@962150028) -> (continuation of Deferred@1201707748 after retry RPC@878468493) -> (continuation of Deferred@1758687597 after retry RPC@1043720655))) after lookup
19:58:58.476 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:58:58.477 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] Successfully connected to peer
19:58:58.498 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x526dca06, L:/127.0.0.1:60124 - R:/127.9.144.1:34195] using TOKEN/null
19:58:58.519 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.144.2:39377
19:58:58.520 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.521440  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/logs
19:58:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.2:39377
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.2
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36739
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.2
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:58:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:58.832494 10747 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:58:58.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:58.832758 10747 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:58:58.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:58.832836 10747 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:58:58.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:58.841079 10747 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:58:58.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:58.841219 10747 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.2
19:58:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.850694 10747 tablet_server_runner.cc:78] Tablet server non-default flags:
19:58:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:58:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:58:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.2:39377
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.2
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36739
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.2
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/logs
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:58:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:58:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.852147 10747 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:58:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.853860 10747 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:58:58.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:58.862718 10754 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:58.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:58.862958 10755 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:58.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:58:58.863158 10757 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:58:58.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.864168 10747 server_base.cc:1061] running on GCE node
19:58:58.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.864665 10747 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:58:58.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.865522 10747 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:58:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.866729 10747 hybrid_clock.cc:648] HybridClock initialized: now 1783540738866648 us; error 74 us; skew 500 ppm
19:58:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.867040 10747 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:58:58.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.868979 10747 webserver.cc:533] Webserver started at http://127.9.144.2:36739/ using document root <none> and password file <none>
19:58:58.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.869710 10747 fs_manager.cc:362] Metadata directory not provided
19:58:58.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.869824 10747 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:58:58.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.874759 10747 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.001s	sys 0.003s
19:58:58.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.877516 10763 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:58:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.878899 10747 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
19:58:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.879042 10747 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:58:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d387b01aa6714e8893e35a67daefebb8"
19:58:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:58:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.879734 10747 fs_report.cc:389] FS layout report
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/data
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:58:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:58:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.905196 10747 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:58:58.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.906111 10747 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:58:58.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.906423 10747 kserver.cc:163] Server-wide thread pool size limit: 3276
19:58:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.907437 10747 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:58:58.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.909528 10770 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
19:58:58.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.915169 10747 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
19:58:58.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.915253 10747 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.000s	sys 0.000s
19:58:58.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.915388 10747 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
19:58:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.918363 10747 ts_tablet_manager.cc:616] Registered 1 tablets
19:58:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.918458 10747 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.002s	sys 0.000s
19:58:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.918776 10770 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Bootstrap starting.
19:58:58.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.940330 10770 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Log is configured to *not* fsync() on all Append() calls
19:58:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.956751 10770 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:58:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.957420 10770 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Bootstrap complete.
19:58:58.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.958591 10770 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Time spent bootstrapping tablet: real 0.040s	user 0.020s	sys 0.018s
19:58:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.961971 10747 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.2:39377
19:58:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.962218 10877 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.2:39377 every 8 connection(s)
19:58:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.963713 10747 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:58:58.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.963872 10770 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:58.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.964763 10770 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: d387b01aa6714e8893e35a67daefebb8, State: Initialized, Role: FOLLOWER
19:58:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.965853 10770 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:58:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.972268  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 10747
19:58:58.972 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:58:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.974193 10770 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Time spent starting tablet: real 0.015s	user 0.010s	sys 0.005s
19:58:58.988 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:58:58.989 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.144.1:34195
19:58:58.990 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

19:58:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.991058  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 10072
19:58:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.998572 10878 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:58:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.999002 10878 heartbeater.cc:461] Registering TS with master...
19:58:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:58.999987 10878 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:58:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.003276  9833 ts_manager.cc:194] Re-registered known tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:58:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.004834  9833 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.2:48275
19:58:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.006942 10194 generic_service.cc:196] Checking for leaks (request via RPC)
19:58:59.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.008149 10878 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:58:59.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.010178 10879 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:58:59.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.010316 10879 heartbeater.cc:461] Registering TS with master...
19:58:59.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.010578 10879 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:58:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.011904  9975 ts_manager.cc:194] Re-registered known tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:58:59.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.014703 10883 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:58:59.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.014859 10883 heartbeater.cc:461] Registering TS with master...
19:58:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.015164 10883 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:58:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.016289  9905 ts_manager.cc:194] Re-registered known tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:58:59.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:58:59.106741  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 10072
19:58:59.122 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] handling channelInactive
19:58:59.122 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state READY due to: connection closed
19:58:59.123 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:58:59.124 [DEBUG - Time-limited test] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:58:59.125 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.125 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.125 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.132 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:59.133 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:59.133 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@564506540(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@30cf51c6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96fa46 -> wakeup thread Time-limited test)) after lookup
19:58:59.134 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:58:59.135 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.135 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.135 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.154 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:59.154 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:59.155 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [10ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [12ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [12ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@564506540(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@30cf51c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1069961647 after retry RPC@1424611547), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96fa46 -> wakeup thread Time-limited test -> (continuation of Deferred@1069961647 after retry RPC@1424611547))) after lookup
19:58:59.155 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:58:59.157 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.157 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.157 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.173 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:59.173 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:59.174 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=50), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [10ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [12ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [12ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [32ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [34ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [34ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [50ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@564506540(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@30cf51c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1069961647 after retry RPC@1424611547) -> (continuation of Deferred@1964322125 after retry RPC@1864371252), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96fa46 -> wakeup thread Time-limited test -> (continuation of Deferred@1069961647 after retry RPC@1424611547) -> (continuation of Deferred@1964322125 after retry RPC@1864371252))) after lookup
19:58:59.174 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:58:59.175 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.175 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.175 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.214 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:59.214 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:59.214 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=91), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [10ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [12ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [12ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [32ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [34ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [34ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [50ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [51ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [52ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [52ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [91ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@564506540(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@30cf51c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1069961647 after retry RPC@1424611547) -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@1603883540 after retry RPC@817345663), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96fa46 -> wakeup thread Time-limited test -> (continuation of Deferred@1069961647 after retry RPC@1424611547) -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@1603883540 after retry RPC@817345663))) after lookup
19:58:59.215 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:58:59.216 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.216 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.217 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.453 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:58:59.454 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:58:59.454 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=330), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [10ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [12ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [12ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [32ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [34ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [34ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [50ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [51ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [52ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [52ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [91ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [92ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [93ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [94ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [327ms] refreshing cache from master, [327ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [330ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@564506540(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@30cf51c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1069961647 after retry RPC@1424611547) -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@1603883540 after retry RPC@817345663) -> (continuation of Deferred@2135664883 after retry RPC@1926426409), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96fa46 -> wakeup thread Time-limited test -> (continuation of Deferred@1069961647 after retry RPC@1424611547) -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@1603883540 after retry RPC@817345663) -> (continuation of Deferred@2135664883 after retry RPC@1926426409))) after lookup
19:58:59.455 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:58:59.456 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.456 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:58:59.456 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:00.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.338691 10892 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:59:00.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.338949 10892 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.340633 10892 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:00.341866 10767 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:00.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:00.343348 10767 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:00.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.347572 10654 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "d387b01aa6714e8893e35a67daefebb8" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "218dc41941a04327b46d9ca11df80df0" is_pre_election: true
19:59:00.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.348322 10765 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0, 2d2fa818193a44629aea25a754e9ec74
19:59:00.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.348701 10892 raft_consensus.cc:2749] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
19:59:00.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.459014 10897 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 2d2fa818193a44629aea25a754e9ec74)
19:59:00.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.459194 10897 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:00.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.459858 10897 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:00.462524 10589 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.467478 10832 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8" is_pre_election: true
19:59:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.467808 10832 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 218dc41941a04327b46d9ca11df80df0 in term 2.
19:59:00.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.468664 10589 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, d387b01aa6714e8893e35a67daefebb8; no voters: 2d2fa818193a44629aea25a754e9ec74
19:59:00.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.468935 10897 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 3 FOLLOWER]: Leader pre-election won for term 4
19:59:00.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.469058 10897 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 2d2fa818193a44629aea25a754e9ec74)
19:59:00.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.469125 10897 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 3 FOLLOWER]: Advancing to term 4
19:59:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.471079 10897 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.471585 10897 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 4 election: Requested vote from peers d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.472005 10832 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8"
19:59:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.472164 10832 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 2 FOLLOWER]: Advancing to term 4
19:59:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:00.473037 10589 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.475873 10832 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 218dc41941a04327b46d9ca11df80df0 in term 4.
19:59:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.476272 10589 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, d387b01aa6714e8893e35a67daefebb8; no voters: 2d2fa818193a44629aea25a754e9ec74
19:59:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.476533 10897 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 4 FOLLOWER]: Leader election won for term 4
19:59:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.476914 10897 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 4 LEADER]: Becoming Leader. State: Replica: 218dc41941a04327b46d9ca11df80df0, State: Running, Role: LEADER
19:59:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.477604 10897 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.481004  9833 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 reported cstate change: term changed from 3 to 4, leader changed from 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1) to 218dc41941a04327b46d9ca11df80df0 (127.9.144.3). New cstate: current_term: 4 leader_uuid: "218dc41941a04327b46d9ca11df80df0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: UNKNOWN } } }
19:59:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.881915 10832 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 4 FOLLOWER]: Refusing update from remote peer 218dc41941a04327b46d9ca11df80df0: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
19:59:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:00.882848 10897 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
19:59:00.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:00.900957 10589 consensus_peers.cc:597] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 -> Peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Couldn't send request to peer 2d2fa818193a44629aea25a754e9ec74. Status: Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:59:01.073 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:01.074 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:01.074 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1950), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [9ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [10ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [12ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [12ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [31ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [32ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [34ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [34ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [50ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [51ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [52ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [52ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [88ms] refreshing cache from master, [88ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [91ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [92ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [93ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [94ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [327ms] refreshing cache from master, [327ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [330ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [331ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [333ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [333ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [1947ms] refreshing cache from master, [1948ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [1950ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@564506540(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@30cf51c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1069961647 after retry RPC@1424611547) -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@1603883540 after retry RPC@817345663) -> (continuation of Deferred@2135664883 after retry RPC@1926426409) -> (continuation of Deferred@363126354 after retry RPC@1465584654), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96fa46 -> wakeup thread Time-limited test -> (continuation of Deferred@1069961647 after retry RPC@1424611547) -> (continuation of Deferred@1964322125 after retry RPC@1864371252) -> (continuation of Deferred@1603883540 after retry RPC@817345663) -> (continuation of Deferred@2135664883 after retry RPC@1926426409) -> (continuation of Deferred@363126354 after retry RPC@1465584654))) after lookup
19:59:01.075 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:01.076 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] Successfully connected to peer
19:59:01.091 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xe5957389, L:/127.0.0.1:52100 - R:/127.9.144.3:43859] using TOKEN/null
19:59:01.125 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.144.1:34195
19:59:01.126 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

19:59:01.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.126690  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:01.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:01.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/logs
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.1:34195
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.1
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40347
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.1
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:59:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:01.442256 10925 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:59:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:01.442544 10925 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:01.442631 10925 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:59:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:01.451424 10925 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:59:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:01.451591 10925 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.1
19:59:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.461529 10925 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.1:34195
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.1
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40347
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.1
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/logs
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:59:01.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:01.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:59:01.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:59:01.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:01.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.463145 10925 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:01.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.464857 10925 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:59:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:01.473716 10930 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:01.474453 10931 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:01.475117 10933 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.475440 10925 server_base.cc:1061] running on GCE node
19:59:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.475988 10925 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.476845 10925 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.478052 10925 hybrid_clock.cc:648] HybridClock initialized: now 1783540741477997 us; error 42 us; skew 500 ppm
19:59:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.478376 10925 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:01.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.480400 10925 webserver.cc:533] Webserver started at http://127.9.144.1:40347/ using document root <none> and password file <none>
19:59:01.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.481163 10925 fs_manager.cc:362] Metadata directory not provided
19:59:01.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.481285 10925 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:01.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.486465 10925 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.004s	sys 0.000s
19:59:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.489352 10939 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:59:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.490610 10925 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
19:59:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.490765 10925 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d2fa818193a44629aea25a754e9ec74"
19:59:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:49 on dist-test-slave-trb6"
19:59:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.491493 10925 fs_report.cc:389] FS layout report
19:59:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/data
19:59:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.509725 10925 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:59:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.510640 10925 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.510963 10925 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:01.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.511943 10925 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:59:01.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.514102 10946 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
19:59:01.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.519737 10925 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
19:59:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.519831 10925 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
19:59:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.519976 10925 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
19:59:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.522974 10925 ts_tablet_manager.cc:616] Registered 1 tablets
19:59:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.523072 10925 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.001s	sys 0.000s
19:59:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.523465 10946 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Bootstrap starting.
19:59:01.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.545701 10946 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Log is configured to *not* fsync() on all Append() calls
19:59:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.561908 10925 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.1:34195
19:59:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.561951 11053 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.1:34195 every 8 connection(s)
19:59:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.563252 10925 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:59:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.565652 10946 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:59:01.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.571408  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 10925
19:59:01.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.583742 10946 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Bootstrap complete.
19:59:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.588270 10946 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Time spent bootstrapping tablet: real 0.065s	user 0.023s	sys 0.021s
19:59:01.592 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:59:01.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.601022 10946 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:01.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.602007 10946 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2d2fa818193a44629aea25a754e9ec74, State: Initialized, Role: FOLLOWER
19:59:01.603 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.144.3:43859
19:59:01.603 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

19:59:01.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.604038  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 10571
19:59:01.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.605070 10946 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.613262 10946 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Time spent starting tablet: real 0.025s	user 0.004s	sys 0.008s
19:59:01.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.635589 11056 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:59:01.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.635995 11056 heartbeater.cc:461] Registering TS with master...
19:59:01.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.636215 11055 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:59:01.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.636366 11055 heartbeater.cc:461] Registering TS with master...
19:59:01.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.637197 11056 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:59:01.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.637341 11055 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:59:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.639183 11054 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:59:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.639325 11054 heartbeater.cc:461] Registering TS with master...
19:59:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.639838 11054 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:59:01.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.640664  9975 ts_manager.cc:194] Re-registered known tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:01.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.640980  9833 ts_manager.cc:194] Re-registered known tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:01.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.641091  9905 ts_manager.cc:194] Re-registered known tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:01.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.642517  9833 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.1:44913
19:59:01.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.645273 11054 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:59:01.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.645646 10695 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:01.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:01.742497  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 10571
19:59:01.757 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:01.758 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] handling channelInactive
19:59:01.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state READY due to: connection closed
19:59:01.771 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:01.772 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.772 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.772 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.793 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:01.794 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:01.794 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [14ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1909175266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@7e7ffaa9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22f3e74 -> wakeup thread Time-limited test)) after lookup
19:59:01.794 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:01.795 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.795 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.795 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.813 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:01.813 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:01.814 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=55), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [14ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [36ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [37ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [37ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1909175266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@7e7ffaa9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@418909236 after retry RPC@1667194305), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22f3e74 -> wakeup thread Time-limited test -> (continuation of Deferred@418909236 after retry RPC@1667194305))) after lookup
19:59:01.815 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:01.816 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.816 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.816 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.834 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:01.834 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:01.834 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=76), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [14ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [36ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [37ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [37ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [56ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [58ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [58ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [72ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [75ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1909175266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@7e7ffaa9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@418909236 after retry RPC@1667194305) -> (continuation of Deferred@1342358074 after retry RPC@1235889326), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22f3e74 -> wakeup thread Time-limited test -> (continuation of Deferred@418909236 after retry RPC@1667194305) -> (continuation of Deferred@1342358074 after retry RPC@1235889326))) after lookup
19:59:01.835 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:01.836 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.836 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.836 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.893 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:01.893 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:01.894 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=135), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [14ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [36ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [37ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [37ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [56ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [58ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [58ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [72ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [75ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [77ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [78ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [78ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [135ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1909175266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@7e7ffaa9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@418909236 after retry RPC@1667194305) -> (continuation of Deferred@1342358074 after retry RPC@1235889326) -> (continuation of Deferred@1559157245 after retry RPC@1144603026), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22f3e74 -> wakeup thread Time-limited test -> (continuation of Deferred@418909236 after retry RPC@1667194305) -> (continuation of Deferred@1342358074 after retry RPC@1235889326) -> (continuation of Deferred@1559157245 after retry RPC@1144603026))) after lookup
19:59:01.895 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:01.896 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.896 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:01.896 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:02.633 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:02.633 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:02.633 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=875), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [14ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [36ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [37ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [37ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [56ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [58ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [58ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [72ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [75ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [77ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [78ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [78ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [135ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [136ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [138ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [138ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [872ms] refreshing cache from master, [872ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [874ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1909175266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@7e7ffaa9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@418909236 after retry RPC@1667194305) -> (continuation of Deferred@1342358074 after retry RPC@1235889326) -> (continuation of Deferred@1559157245 after retry RPC@1144603026) -> (continuation of Deferred@1942470659 after retry RPC@2105289272), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22f3e74 -> wakeup thread Time-limited test -> (continuation of Deferred@418909236 after retry RPC@1667194305) -> (continuation of Deferred@1342358074 after retry RPC@1235889326) -> (continuation of Deferred@1559157245 after retry RPC@1144603026) -> (continuation of Deferred@1942470659 after retry RPC@2105289272))) after lookup
19:59:02.634 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:02.635 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:02.635 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:02.636 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:02.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:02.754925 11069 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:59:02.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:02.755173 11069 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:02.757155 11069 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:02.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:02.758502 10941 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:02.759843 10941 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:02.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:02.764349 10832 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "2d2fa818193a44629aea25a754e9ec74" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8" is_pre_election: true
19:59:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:02.765180 10943 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2d2fa818193a44629aea25a754e9ec74; no voters: 218dc41941a04327b46d9ca11df80df0, d387b01aa6714e8893e35a67daefebb8
19:59:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:02.765549 11069 raft_consensus.cc:2749] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:03.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.132606 11074 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 218dc41941a04327b46d9ca11df80df0)
19:59:03.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.132793 11074 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:03.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.133466 11074 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:03.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:03.135908 10765 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:03.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.141408 11008 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "d387b01aa6714e8893e35a67daefebb8" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74" is_pre_election: true
19:59:03.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.141769 11008 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d387b01aa6714e8893e35a67daefebb8 in term 3.
19:59:03.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.142576 10767 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2d2fa818193a44629aea25a754e9ec74, d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0
19:59:03.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.142858 11074 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 4 FOLLOWER]: Leader pre-election won for term 5
19:59:03.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.142968 11074 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 218dc41941a04327b46d9ca11df80df0)
19:59:03.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.143067 11074 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 4 FOLLOWER]: Advancing to term 5
19:59:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.145133 11074 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.145689 11074 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 5 election: Requested vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.146198 11008 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "d387b01aa6714e8893e35a67daefebb8" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74"
19:59:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.146382 11008 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 3 FOLLOWER]: Advancing to term 5
19:59:03.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:03.147038 10765 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:03.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.150116 11008 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d387b01aa6714e8893e35a67daefebb8 in term 5.
19:59:03.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.150506 10767 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2d2fa818193a44629aea25a754e9ec74, d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0
19:59:03.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.150765 11074 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 5 FOLLOWER]: Leader election won for term 5
19:59:03.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.151209 11074 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 5 LEADER]: Becoming Leader. State: Replica: d387b01aa6714e8893e35a67daefebb8, State: Running, Role: LEADER
19:59:03.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.151700 11074 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.155069  9833 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 reported cstate change: term changed from 4 to 5, leader changed from 218dc41941a04327b46d9ca11df80df0 (127.9.144.3) to d387b01aa6714e8893e35a67daefebb8 (127.9.144.2). New cstate: current_term: 5 leader_uuid: "d387b01aa6714e8893e35a67daefebb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: UNKNOWN } } }
19:59:03.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.585976 11008 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 5 FOLLOWER]: Refusing update from remote peer d387b01aa6714e8893e35a67daefebb8: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
19:59:03.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:03.586778 11074 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
19:59:03.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:03.598965 10765 consensus_peers.cc:597] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 -> Peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Couldn't send request to peer 218dc41941a04327b46d9ca11df80df0. Status: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:59:05.073 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:05.074 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:05.074 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=13, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=3316), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [14ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [14ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [35ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [36ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [37ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [37ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [56ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [58ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [58ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [72ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [75ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [77ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [78ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [78ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [135ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [136ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [138ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [138ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [872ms] refreshing cache from master, [872ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [874ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [876ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [877ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [878ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [3313ms] refreshing cache from master, [3313ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [3315ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1909175266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@7e7ffaa9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@418909236 after retry RPC@1667194305) -> (continuation of Deferred@1342358074 after retry RPC@1235889326) -> (continuation of Deferred@1559157245 after retry RPC@1144603026) -> (continuation of Deferred@1942470659 after retry RPC@2105289272) -> (continuation of Deferred@1769205458 after retry RPC@1967268910), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22f3e74 -> wakeup thread Time-limited test -> (continuation of Deferred@418909236 after retry RPC@1667194305) -> (continuation of Deferred@1342358074 after retry RPC@1235889326) -> (continuation of Deferred@1559157245 after retry RPC@1144603026) -> (continuation of Deferred@1942470659 after retry RPC@2105289272) -> (continuation of Deferred@1769205458 after retry RPC@1967268910))) after lookup
19:59:05.075 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:05.076 [DEBUG - kudu-nio-0] (Connection.java:757) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] Successfully connected to peer
19:59:05.091 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xfd8d71a7, L:/127.0.0.1:39526 - R:/127.9.144.2:39377] using TOKEN/null
19:59:05.115 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.144.3:43859
19:59:05.115 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.116147  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/logs
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.3:43859
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.3
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37285
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.3
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:59:05.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:05.411310 10765 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111) [suppressed 6 similar messages]
19:59:05.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:05.430217 11110 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:59:05.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:05.430521 11110 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:05.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:05.430636 11110 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:59:05.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:05.438897 11110 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:59:05.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:05.439122 11110 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.3
19:59:05.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.449447 11110 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:05.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.3:43859
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.3
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37285
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.3
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/logs
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:05.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:59:05.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:05.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:59:05.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:59:05.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:05.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.451588 11110 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:05.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.453421 11110 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:59:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:05.462428 11115 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:05.462571 11116 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:05.462761 11118 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.463495 11110 server_base.cc:1061] running on GCE node
19:59:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.463972 11110 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.464782 11110 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:05.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.465979 11110 hybrid_clock.cc:648] HybridClock initialized: now 1783540745465920 us; error 53 us; skew 500 ppm
19:59:05.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.466317 11110 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.468302 11110 webserver.cc:533] Webserver started at http://127.9.144.3:37285/ using document root <none> and password file <none>
19:59:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.469112 11110 fs_manager.cc:362] Metadata directory not provided
19:59:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.469215 11110 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:05.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.474407 11110 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.001s	sys 0.003s
19:59:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.477146 11124 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:59:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.478413 11110 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.478555 11110 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "218dc41941a04327b46d9ca11df80df0"
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.479282 11110 fs_report.cc:389] FS layout report
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/data
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:05.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:05.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:05.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:05.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.516834 11110 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:59:05.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.517774 11110 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.518086 11110 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:05.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.519042 11110 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:59:05.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.521205 11131 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
19:59:05.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.526700 11110 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
19:59:05.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.526784 11110 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
19:59:05.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.526909 11110 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
19:59:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.529986 11110 ts_tablet_manager.cc:616] Registered 1 tablets
19:59:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.530079 11110 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.002s	sys 0.000s
19:59:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.530462 11131 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap starting.
19:59:05.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.552161 11131 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Log is configured to *not* fsync() on all Append() calls
19:59:05.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.573423 11110 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.3:43859
19:59:05.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.573464 11238 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.3:43859 every 8 connection(s)
19:59:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.574762 11110 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:59:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.575327 11131 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:59:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.575974 11131 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap complete.
19:59:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.577373 11131 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Time spent bootstrapping tablet: real 0.047s	user 0.032s	sys 0.014s
19:59:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.581406  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 11110
19:59:05.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.593915 11131 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.594974 11131 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 218dc41941a04327b46d9ca11df80df0, State: Initialized, Role: FOLLOWER
19:59:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.597955 11131 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:05.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.601765 11239 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:59:05.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.602093 11239 heartbeater.cc:461] Registering TS with master...
19:59:05.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.603189 11239 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:59:05.603 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:59:05.604 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.144.2:39377
19:59:05.605 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

19:59:05.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.605556  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 10747
19:59:05.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.606496 11131 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Time spent starting tablet: real 0.029s	user 0.007s	sys 0.003s
19:59:05.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.613240  9833 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:59:05.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.614766  9833 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.3:55331
19:59:05.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.612141 11243 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:59:05.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.615993 11243 heartbeater.cc:461] Registering TS with master...
19:59:05.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.616569 11243 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:59:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.617990  9905 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:59:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.619676 11240 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:59:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.619822 11240 heartbeater.cc:461] Registering TS with master...
19:59:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.620301 11240 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:59:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.621153 11239 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:59:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.621760  9976 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:59:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.623195 10873 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:05.716266  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 10747
19:59:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:05.730046 10943 connection.cc:573] server connection from 127.9.144.2:48853 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
19:59:05.730 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:05.730 [DEBUG - kudu-nio-0] (Connection.java:239) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] handling channelInactive
19:59:05.731 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state READY due to: connection closed
19:59:05.751 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:05.753 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.753 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.753 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.773 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:05.773 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:05.773 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=3, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [20ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [22ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [22ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@553181682(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@8293820 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7dc66395 -> wakeup thread Time-limited test)) after lookup
19:59:05.774 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:05.775 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.775 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.776 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.792 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:05.792 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:05.793 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=5, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [20ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [22ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [22ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [43ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [44ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [45ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [61ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@553181682(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@8293820 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7dc66395 -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634))) after lookup
19:59:05.793 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:05.794 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.794 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.794 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.834 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:05.834 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:05.834 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=7, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=103), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [20ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [22ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [22ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [43ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [44ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [45ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [61ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [62ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [63ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [63ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [102ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@553181682(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@8293820 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634) -> (continuation of Deferred@1827370618 after retry RPC@1674407147), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7dc66395 -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634) -> (continuation of Deferred@1827370618 after retry RPC@1674407147))) after lookup
19:59:05.835 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:05.836 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.836 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.836 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.993 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:05.993 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:05.994 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=9, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=263), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [20ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [22ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [22ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [43ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [44ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [45ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [61ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [62ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [63ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [63ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [102ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [104ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [105ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [105ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [259ms] refreshing cache from master, [259ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [262ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@553181682(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@8293820 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634) -> (continuation of Deferred@1827370618 after retry RPC@1674407147) -> (continuation of Deferred@865251773 after retry RPC@1001400592), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7dc66395 -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634) -> (continuation of Deferred@1827370618 after retry RPC@1674407147) -> (continuation of Deferred@865251773 after retry RPC@1001400592))) after lookup
19:59:05.994 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:05.995 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.996 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:05.996 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:06.034 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:06.034 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:06.034 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=11, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=303), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [20ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [22ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [22ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [43ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [44ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [45ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [61ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [62ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [63ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [63ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [102ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [104ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [105ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [105ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [259ms] refreshing cache from master, [259ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [262ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [263ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [265ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [265ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [300ms] refreshing cache from master, [300ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [302ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@553181682(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@8293820 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634) -> (continuation of Deferred@1827370618 after retry RPC@1674407147) -> (continuation of Deferred@865251773 after retry RPC@1001400592) -> (continuation of Deferred@1325827991 after retry RPC@1975836622), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7dc66395 -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634) -> (continuation of Deferred@1827370618 after retry RPC@1674407147) -> (continuation of Deferred@865251773 after retry RPC@1001400592) -> (continuation of Deferred@1325827991 after retry RPC@1975836622))) after lookup
19:59:06.035 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:06.036 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:06.036 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:06.037 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:06.933 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:06.933 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:06.934 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=13, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1203), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [20ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [22ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [22ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [43ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [44ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [45ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [61ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [62ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [63ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [63ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [102ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [104ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [105ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [105ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [259ms] refreshing cache from master, [259ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [262ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [263ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [265ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [265ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [300ms] refreshing cache from master, [300ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [302ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [304ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [306ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [306ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [1199ms] refreshing cache from master, [1199ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [1202ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@553181682(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@8293820 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634) -> (continuation of Deferred@1827370618 after retry RPC@1674407147) -> (continuation of Deferred@865251773 after retry RPC@1001400592) -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7dc66395 -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634) -> (continuation of Deferred@1827370618 after retry RPC@1674407147) -> (continuation of Deferred@865251773 after retry RPC@1001400592) -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985))) after lookup
19:59:06.935 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:06.935 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:06.936 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:06.936 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:07.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.217929 11255 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader d387b01aa6714e8893e35a67daefebb8)
19:59:07.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.218112 11255 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:07.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.218700 11255 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:07.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:07.222632 10943 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:59:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.227859 11194 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "2d2fa818193a44629aea25a754e9ec74" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "218dc41941a04327b46d9ca11df80df0" is_pre_election: true
19:59:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.228327 11194 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2d2fa818193a44629aea25a754e9ec74 in term 4.
19:59:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.229228 10941 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, 2d2fa818193a44629aea25a754e9ec74; no voters: d387b01aa6714e8893e35a67daefebb8
19:59:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.229494 11255 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 5 FOLLOWER]: Leader pre-election won for term 6
19:59:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.229605 11255 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 5 FOLLOWER]: Starting leader election (detected failure of leader d387b01aa6714e8893e35a67daefebb8)
19:59:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.229669 11255 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 5 FOLLOWER]: Advancing to term 6
19:59:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.231638 11255 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:07.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.232163 11255 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 6 election: Requested vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:07.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.232584 11194 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "2d2fa818193a44629aea25a754e9ec74" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "218dc41941a04327b46d9ca11df80df0"
19:59:07.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.232757 11194 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 4 FOLLOWER]: Advancing to term 6
19:59:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:07.233445 10943 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:59:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.236316 11194 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2d2fa818193a44629aea25a754e9ec74 in term 6.
19:59:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.236703 10941 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, 2d2fa818193a44629aea25a754e9ec74; no voters: d387b01aa6714e8893e35a67daefebb8
19:59:07.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.236965 11255 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 6 FOLLOWER]: Leader election won for term 6
19:59:07.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.237341 11255 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 6 LEADER]: Becoming Leader. State: Replica: 2d2fa818193a44629aea25a754e9ec74, State: Running, Role: LEADER
19:59:07.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.237798 11255 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:07.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.241278  9833 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 reported cstate change: term changed from 5 to 6, leader changed from d387b01aa6714e8893e35a67daefebb8 (127.9.144.2) to 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1). New cstate: current_term: 6 leader_uuid: "2d2fa818193a44629aea25a754e9ec74" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: HEALTHY } } }
19:59:07.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:07.759083 10943 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111) [suppressed 2 similar messages]
19:59:07.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:07.760229 10943 consensus_peers.cc:597] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 -> Peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Couldn't send request to peer d387b01aa6714e8893e35a67daefebb8. Status: Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:59:07.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.763489 11194 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 6 FOLLOWER]: Refusing update from remote peer 2d2fa818193a44629aea25a754e9ec74: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
19:59:07.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:07.764322 11274 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [LEADER]: Connected to new peer: Peer: permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
19:59:09.253 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:09.254 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:09.255 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=15, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=3523), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [20ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [22ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [22ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [42ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [43ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [44ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [45ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [61ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [62ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [63ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [63ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [102ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [104ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [105ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [105ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [259ms] refreshing cache from master, [259ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [262ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [263ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [265ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [265ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [300ms] refreshing cache from master, [300ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [302ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [304ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [306ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [306ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [1199ms] refreshing cache from master, [1199ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [1202ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [1203ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [1205ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [1205ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [3519ms] refreshing cache from master, [3519ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [3522ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@553181682(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@8293820 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634) -> (continuation of Deferred@1827370618 after retry RPC@1674407147) -> (continuation of Deferred@865251773 after retry RPC@1001400592) -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985) -> (continuation of Deferred@1288604902 after retry RPC@257302969), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7dc66395 -> wakeup thread Time-limited test -> (continuation of Deferred@549989365 after retry RPC@1859923634) -> (continuation of Deferred@1827370618 after retry RPC@1674407147) -> (continuation of Deferred@865251773 after retry RPC@1001400592) -> (continuation of Deferred@1325827991 after retry RPC@1975836622) -> (continuation of Deferred@25932393 after retry RPC@1390556985) -> (continuation of Deferred@1288604902 after retry RPC@257302969))) after lookup
19:59:09.256 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:09.257 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] Successfully connected to peer
19:59:09.276 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x86ce1856, L:/127.0.0.1:49084 - R:/127.9.144.1:34195] using TOKEN/null
19:59:09.301 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.144.2:39377
19:59:09.302 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.303447  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/logs
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:59:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.2:39377
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.2
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36739
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.2
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:59:09.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:09.626710 11295 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:59:09.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:09.627025 11295 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:09.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:09.627105 11295 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:59:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:09.635221 11295 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:59:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:09.635372 11295 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.2
19:59:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.645540 11295 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.2:39377
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.2
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36739
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.2
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/logs
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:59:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:09.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.647120 11295 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:09.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.648773 11295 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:59:09.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:09.657994 11303 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:09.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:09.657969 11300 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:09.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:09.657935 11301 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:09.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.658998 11295 server_base.cc:1061] running on GCE node
19:59:09.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.659502 11295 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.660414 11295 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.661595 11295 hybrid_clock.cc:648] HybridClock initialized: now 1783540749661545 us; error 46 us; skew 500 ppm
19:59:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.661868 11295 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.663735 11295 webserver.cc:533] Webserver started at http://127.9.144.2:36739/ using document root <none> and password file <none>
19:59:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.664464 11295 fs_manager.cc:362] Metadata directory not provided
19:59:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.664556 11295 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.669739 11295 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.001s	sys 0.002s
19:59:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.672477 11309 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:59:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.673877 11295 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
19:59:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.674024 11295 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:59:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d387b01aa6714e8893e35a67daefebb8"
19:59:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:59:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.674749 11295 fs_report.cc:389] FS layout report
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/data
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.704902 11295 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:59:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.705868 11295 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.706182 11295 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:09.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.707167 11295 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:59:09.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.709230 11316 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
19:59:09.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.714915 11295 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
19:59:09.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.715031 11295 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
19:59:09.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.715189 11295 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
19:59:09.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.718735 11295 ts_tablet_manager.cc:616] Registered 1 tablets
19:59:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.718843 11295 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.000s	sys 0.002s
19:59:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.719206 11316 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Bootstrap starting.
19:59:09.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.740628 11316 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Log is configured to *not* fsync() on all Append() calls
19:59:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.759920 11295 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.2:39377
19:59:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.759948 11423 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.2:39377 every 8 connection(s)
19:59:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.761449 11295 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:59:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.768297 11316 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:59:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.768935 11316 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Bootstrap complete.
19:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.770262  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 11295
19:59:09.771 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:09.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.777070 11316 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Time spent bootstrapping tablet: real 0.058s	user 0.031s	sys 0.020s
19:59:09.787 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:59:09.788 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.144.1:34195
19:59:09.788 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

19:59:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.788668  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 10925
19:59:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.798970 11316 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.804596 11316 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: d387b01aa6714e8893e35a67daefebb8, State: Initialized, Role: FOLLOWER
19:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.805408 11316 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.806867 11424 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:59:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.807273 11424 heartbeater.cc:461] Registering TS with master...
19:59:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.808090 11316 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Time spent starting tablet: real 0.031s	user 0.008s	sys 0.000s
19:59:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.808429 11424 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:59:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.809012 11426 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:59:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.809155 11426 heartbeater.cc:461] Registering TS with master...
19:59:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.809651 11426 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:59:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.811509  9833 ts_manager.cc:194] Re-registered known tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.812654  9833 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.2:51437
19:59:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.813745  9905 ts_manager.cc:194] Re-registered known tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.818322 11424 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:59:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.819509 11425 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:59:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.819638 11425 heartbeater.cc:461] Registering TS with master...
19:59:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.819954 11425 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:59:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.820437 11049 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.821621  9976 ts_manager.cc:194] Re-registered known tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:09.918975  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 10925
19:59:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:09.933365 11128 connection.cc:573] server connection from 127.9.144.1:36587 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
19:59:09.933 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] handling channelInactive
19:59:09.933 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state READY due to: connection closed
19:59:09.934 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:09.934 [DEBUG - Time-limited test] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:09.935 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.935 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.936 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.953 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:09.953 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:09.953 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1064030193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@2ebe9d16 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c1dc501 -> wakeup thread Time-limited test)) after lookup
19:59:09.955 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:09.956 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.956 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.956 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.973 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:09.973 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:09.973 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [20ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [22ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1064030193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@2ebe9d16 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c1dc501 -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314))) after lookup
19:59:09.974 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:09.974 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.974 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.975 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.993 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:09.993 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:09.993 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [20ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [22ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [39ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [41ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [41ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [59ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1064030193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@2ebe9d16 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314) -> (continuation of Deferred@593838227 after retry RPC@1249932118), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c1dc501 -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314) -> (continuation of Deferred@593838227 after retry RPC@1249932118))) after lookup
19:59:09.994 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:09.995 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.995 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:09.995 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:10.073 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:10.074 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:10.074 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=140), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [20ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [22ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [39ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [41ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [41ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [59ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [60ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [61ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [61ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [136ms] refreshing cache from master, [136ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [139ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1064030193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@2ebe9d16 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314) -> (continuation of Deferred@593838227 after retry RPC@1249932118) -> (continuation of Deferred@570526670 after retry RPC@1808946715), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c1dc501 -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314) -> (continuation of Deferred@593838227 after retry RPC@1249932118) -> (continuation of Deferred@570526670 after retry RPC@1808946715))) after lookup
19:59:10.075 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:10.076 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:10.076 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:10.076 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:10.233 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:10.234 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:10.234 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=300), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [20ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [22ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [39ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [41ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [41ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [59ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [60ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [61ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [61ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [136ms] refreshing cache from master, [136ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [139ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [141ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [142ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [142ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [296ms] refreshing cache from master, [297ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [299ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1064030193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@2ebe9d16 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314) -> (continuation of Deferred@593838227 after retry RPC@1249932118) -> (continuation of Deferred@570526670 after retry RPC@1808946715) -> (continuation of Deferred@488729006 after retry RPC@1681557151), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c1dc501 -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314) -> (continuation of Deferred@593838227 after retry RPC@1249932118) -> (continuation of Deferred@570526670 after retry RPC@1808946715) -> (continuation of Deferred@488729006 after retry RPC@1681557151))) after lookup
19:59:10.235 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:10.236 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:10.236 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:10.236 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:10.733 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:10.734 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:10.734 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=800), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [20ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [22ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [39ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [41ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [41ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [59ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [60ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [61ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [61ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [136ms] refreshing cache from master, [136ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [139ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [141ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [142ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [142ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [296ms] refreshing cache from master, [297ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [299ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [301ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [302ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [302ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [796ms] refreshing cache from master, [797ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [799ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1064030193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@2ebe9d16 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314) -> (continuation of Deferred@593838227 after retry RPC@1249932118) -> (continuation of Deferred@570526670 after retry RPC@1808946715) -> (continuation of Deferred@488729006 after retry RPC@1681557151) -> (continuation of Deferred@829306776 after retry RPC@30425428), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c1dc501 -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314) -> (continuation of Deferred@593838227 after retry RPC@1249932118) -> (continuation of Deferred@570526670 after retry RPC@1808946715) -> (continuation of Deferred@488729006 after retry RPC@1681557151) -> (continuation of Deferred@829306776 after retry RPC@30425428))) after lookup
19:59:10.735 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:10.736 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:10.736 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:10.736 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:11.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.373167 11439 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:59:11.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.373409 11439 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.374910 11439 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:11.376571 11313 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:11.377916 11313 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:11.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.390674 11194 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "d387b01aa6714e8893e35a67daefebb8" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "218dc41941a04327b46d9ca11df80df0" is_pre_election: true
19:59:11.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.391674 11311 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0, 2d2fa818193a44629aea25a754e9ec74
19:59:11.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.392143 11439 raft_consensus.cc:2749] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
19:59:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.420457 11444 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 2d2fa818193a44629aea25a754e9ec74)
19:59:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.420680 11444 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:11.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.422359 11444 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:11.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:11.424382 11128 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:11.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:11.427170 11128 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.431345 11379 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8" is_pre_election: true
19:59:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.431738 11379 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 218dc41941a04327b46d9ca11df80df0 in term 5.
19:59:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.432672 11128 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, d387b01aa6714e8893e35a67daefebb8; no voters: 2d2fa818193a44629aea25a754e9ec74
19:59:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.433100 11444 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 6 FOLLOWER]: Leader pre-election won for term 7
19:59:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.433230 11444 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 2d2fa818193a44629aea25a754e9ec74)
19:59:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.433313 11444 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 6 FOLLOWER]: Advancing to term 7
19:59:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.436456 11444 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.437292 11444 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 7 election: Requested vote from peers d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.437742 11379 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8"
19:59:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.437912 11379 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 5 FOLLOWER]: Advancing to term 7
19:59:11.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:11.441015 11128 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.442122 11379 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 218dc41941a04327b46d9ca11df80df0 in term 7.
19:59:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.442545 11128 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, d387b01aa6714e8893e35a67daefebb8; no voters: 2d2fa818193a44629aea25a754e9ec74
19:59:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.442907 11444 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 7 FOLLOWER]: Leader election won for term 7
19:59:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.443367 11444 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 7 LEADER]: Becoming Leader. State: Replica: 218dc41941a04327b46d9ca11df80df0, State: Running, Role: LEADER
19:59:11.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.443933 11444 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:11.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.447728  9833 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 reported cstate change: term changed from 6 to 7, leader changed from 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1) to 218dc41941a04327b46d9ca11df80df0 (127.9.144.3). New cstate: current_term: 7 leader_uuid: "218dc41941a04327b46d9ca11df80df0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: UNKNOWN } } }
19:59:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:11.926803 11128 consensus_peers.cc:597] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 -> Peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Couldn't send request to peer 2d2fa818193a44629aea25a754e9ec74. Status: Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:59:11.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.941557 11379 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 7 FOLLOWER]: Refusing update from remote peer 218dc41941a04327b46d9ca11df80df0: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
19:59:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:11.942413 11444 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
19:59:12.374 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:12.374 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:12.375 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2440), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [19ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [20ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [22ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [38ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [39ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [41ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [41ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [59ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [60ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [61ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [61ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [136ms] refreshing cache from master, [136ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [139ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [141ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [142ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [142ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [296ms] refreshing cache from master, [297ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [299ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [301ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [302ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [302ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [796ms] refreshing cache from master, [797ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [799ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [801ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [802ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [802ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2437ms] refreshing cache from master, [2437ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [2440ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1064030193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@2ebe9d16 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314) -> (continuation of Deferred@593838227 after retry RPC@1249932118) -> (continuation of Deferred@570526670 after retry RPC@1808946715) -> (continuation of Deferred@488729006 after retry RPC@1681557151) -> (continuation of Deferred@829306776 after retry RPC@30425428) -> (continuation of Deferred@644370228 after retry RPC@632381516), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c1dc501 -> wakeup thread Time-limited test -> (continuation of Deferred@1041454092 after retry RPC@1280621314) -> (continuation of Deferred@593838227 after retry RPC@1249932118) -> (continuation of Deferred@570526670 after retry RPC@1808946715) -> (continuation of Deferred@488729006 after retry RPC@1681557151) -> (continuation of Deferred@829306776 after retry RPC@30425428) -> (continuation of Deferred@644370228 after retry RPC@632381516))) after lookup
19:59:12.375 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:12.376 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] Successfully connected to peer
19:59:12.395 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xf7a44884, L:/127.0.0.1:35818 - R:/127.9.144.3:43859] using TOKEN/null
19:59:12.415 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.144.1:34195
19:59:12.416 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.416916  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/logs
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.1:34195
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.1
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40347
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.1
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:59:12.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:12.728854 11472 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:59:12.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:12.729215 11472 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:12.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:12.729334 11472 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:59:12.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:12.737707 11472 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:59:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:12.737906 11472 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.1
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.748147 11472 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.1:34195
19:59:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.1
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40347
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.1
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/logs
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:59:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:12.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.750352 11472 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.752278 11472 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:59:12.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:12.761217 11478 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:12.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:12.761262 11477 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:12.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:12.762444 11480 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:12.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.763458 11472 server_base.cc:1061] running on GCE node
19:59:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.764097 11472 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.765216 11472 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.766430 11472 hybrid_clock.cc:648] HybridClock initialized: now 1783540752766359 us; error 58 us; skew 500 ppm
19:59:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.766788 11472 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:12.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.769407 11472 webserver.cc:533] Webserver started at http://127.9.144.1:40347/ using document root <none> and password file <none>
19:59:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.770334 11472 fs_manager.cc:362] Metadata directory not provided
19:59:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.770447 11472 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:12.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.776669 11472 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
19:59:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.779496 11486 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:59:12.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.780920 11472 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.002s
19:59:12.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.781080 11472 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:12.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d2fa818193a44629aea25a754e9ec74"
19:59:12.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:49 on dist-test-slave-trb6"
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.781872 11472 fs_report.cc:389] FS layout report
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/data
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.803941 11472 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:59:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.804708 11472 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.804952 11472 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.805855 11472 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:59:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.807785 11493 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
19:59:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.813557 11472 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
19:59:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.813637 11472 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
19:59:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.813732 11472 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
19:59:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.816594 11472 ts_tablet_manager.cc:616] Registered 1 tablets
19:59:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.816676 11472 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
19:59:12.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.817152 11493 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Bootstrap starting.
19:59:12.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.841176 11493 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Log is configured to *not* fsync() on all Append() calls
19:59:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.857817 11472 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.1:34195
19:59:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.857867 11600 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.1:34195 every 8 connection(s)
19:59:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.859342 11472 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:59:12.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.863423  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 11472
19:59:12.864 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:12.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.870793 11493 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:59:12.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.871765 11493 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Bootstrap complete.
19:59:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.873206 11493 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Time spent bootstrapping tablet: real 0.056s	user 0.044s	sys 0.011s
19:59:12.879 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:59:12.880 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.144.3:43859
19:59:12.880 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

19:59:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.879683 11493 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.880669 11493 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2d2fa818193a44629aea25a754e9ec74, State: Initialized, Role: FOLLOWER
19:59:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.881354  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 11110
19:59:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.881556 11493 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:12.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.898599 11603 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:59:12.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.899019 11603 heartbeater.cc:461] Registering TS with master...
19:59:12.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.900141 11603 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:59:12.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.903841  9905 ts_manager.cc:194] Re-registered known tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:12.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.904130 11602 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:59:12.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.904268 11602 heartbeater.cc:461] Registering TS with master...
19:59:12.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.904608 11493 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Time spent starting tablet: real 0.031s	user 0.012s	sys 0.017s
19:59:12.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.904816 11602 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:59:12.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.905608 11234 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:12.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.906997  9975 ts_manager.cc:194] Re-registered known tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.910355 11601 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:59:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.910480 11601 heartbeater.cc:461] Registering TS with master...
19:59:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.910916 11601 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:59:12.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.912122  9834 ts_manager.cc:194] Re-registered known tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:12.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.913506  9834 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.1:47543
19:59:12.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:12.916072 11601 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:59:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.005132  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 11110
19:59:13.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:13.018864  9960 connection.cc:573] server connection from 127.9.144.3:58127 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
19:59:13.019 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] handling channelInactive
19:59:13.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:13.019295  9821 connection.cc:573] server connection from 127.9.144.3:55331 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
19:59:13.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state READY due to: connection closed
19:59:13.019 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:13.020 [DEBUG - Time-limited test] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:13.021 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.021 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.021 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.033 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:13.033 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:13.033 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [1ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [12ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1733814105(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5c62f25a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@33a6a791 -> wakeup thread Time-limited test)) after lookup
19:59:13.034 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:13.035 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.035 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.035 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.054 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:13.054 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:13.055 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [1ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [12ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [15ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1733814105(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5c62f25a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@33a6a791 -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199))) after lookup
19:59:13.055 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:13.056 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.056 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.057 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.093 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:13.093 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:13.094 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=74), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [1ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [12ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [15ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [35ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [37ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [37ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [73ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1733814105(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5c62f25a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199) -> (continuation of Deferred@1522263530 after retry RPC@1314701861), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@33a6a791 -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199) -> (continuation of Deferred@1522263530 after retry RPC@1314701861))) after lookup
19:59:13.095 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:13.096 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.096 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.096 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.193 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:13.193 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:13.193 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=173), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [1ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [12ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [15ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [35ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [37ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [37ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [73ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [74ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [76ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [76ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [173ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1733814105(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5c62f25a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199) -> (continuation of Deferred@1522263530 after retry RPC@1314701861) -> (continuation of Deferred@830033767 after retry RPC@421326471), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@33a6a791 -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199) -> (continuation of Deferred@1522263530 after retry RPC@1314701861) -> (continuation of Deferred@830033767 after retry RPC@421326471))) after lookup
19:59:13.194 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:13.195 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.195 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.195 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.374 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:13.374 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:13.374 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=354), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [1ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [12ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [15ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [35ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [37ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [37ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [73ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [74ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [76ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [76ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [173ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [174ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [175ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [175ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [351ms] refreshing cache from master, [351ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [353ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1733814105(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5c62f25a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199) -> (continuation of Deferred@1522263530 after retry RPC@1314701861) -> (continuation of Deferred@830033767 after retry RPC@421326471) -> (continuation of Deferred@1605791663 after retry RPC@555797000), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@33a6a791 -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199) -> (continuation of Deferred@1522263530 after retry RPC@1314701861) -> (continuation of Deferred@830033767 after retry RPC@421326471) -> (continuation of Deferred@1605791663 after retry RPC@555797000))) after lookup
19:59:13.375 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:13.376 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.376 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.376 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.474 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:13.474 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:13.475 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=454), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [1ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [12ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [15ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [35ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [37ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [37ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [73ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [74ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [76ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [76ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [173ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [174ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [175ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [175ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [351ms] refreshing cache from master, [351ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [353ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [355ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [356ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [356ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [451ms] refreshing cache from master, [451ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [453ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1733814105(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5c62f25a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199) -> (continuation of Deferred@1522263530 after retry RPC@1314701861) -> (continuation of Deferred@830033767 after retry RPC@421326471) -> (continuation of Deferred@1605791663 after retry RPC@555797000) -> (continuation of Deferred@1841406144 after retry RPC@1251362810), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@33a6a791 -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199) -> (continuation of Deferred@1522263530 after retry RPC@1314701861) -> (continuation of Deferred@830033767 after retry RPC@421326471) -> (continuation of Deferred@1605791663 after retry RPC@555797000) -> (continuation of Deferred@1841406144 after retry RPC@1251362810))) after lookup
19:59:13.475 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:13.477 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.477 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.477 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.916270 11616 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 218dc41941a04327b46d9ca11df80df0)
19:59:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.916438 11616 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.917068 11616 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:13.919205 11311 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.924899 11555 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "d387b01aa6714e8893e35a67daefebb8" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74" is_pre_election: true
19:59:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.925347 11555 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d387b01aa6714e8893e35a67daefebb8 in term 6.
19:59:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.926229 11313 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2d2fa818193a44629aea25a754e9ec74, d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0
19:59:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.926530 11616 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 7 FOLLOWER]: Leader pre-election won for term 8
19:59:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.926630 11616 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 218dc41941a04327b46d9ca11df80df0)
19:59:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.926693 11616 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 7 FOLLOWER]: Advancing to term 8
19:59:13.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.928613 11616 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:13.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.929190 11616 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 8 election: Requested vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:13.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.929716 11555 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "d387b01aa6714e8893e35a67daefebb8" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74"
19:59:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.929893 11555 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 6 FOLLOWER]: Advancing to term 8
19:59:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:13.930563 11311 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.933494 11555 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d387b01aa6714e8893e35a67daefebb8 in term 8.
19:59:13.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.933887 11313 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2d2fa818193a44629aea25a754e9ec74, d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0
19:59:13.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.934168 11616 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 8 FOLLOWER]: Leader election won for term 8
19:59:13.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.934597 11616 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 8 LEADER]: Becoming Leader. State: Replica: d387b01aa6714e8893e35a67daefebb8, State: Running, Role: LEADER
19:59:13.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.935108 11616 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:13.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:13.938881  9834 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 reported cstate change: term changed from 7 to 8, leader changed from 218dc41941a04327b46d9ca11df80df0 (127.9.144.3) to d387b01aa6714e8893e35a67daefebb8 (127.9.144.2). New cstate: current_term: 8 leader_uuid: "d387b01aa6714e8893e35a67daefebb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: UNKNOWN } } }
19:59:14.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:14.416800 11555 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 8 FOLLOWER]: Refusing update from remote peer d387b01aa6714e8893e35a67daefebb8: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
19:59:14.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:14.417830 11616 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
19:59:14.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:14.431324 11311 consensus_peers.cc:597] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 -> Peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Couldn't send request to peer 218dc41941a04327b46d9ca11df80df0. Status: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:59:16.214 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:16.214 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:16.214 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=3194), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [1ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [12ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [15ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [15ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [34ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [35ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [37ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [37ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [73ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [74ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [76ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [76ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [173ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [174ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [175ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [175ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [351ms] refreshing cache from master, [351ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [353ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [355ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [356ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [356ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [451ms] refreshing cache from master, [451ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [453ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [455ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [457ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [457ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [3190ms] refreshing cache from master, [3190ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [3193ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1733814105(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5c62f25a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199) -> (continuation of Deferred@1522263530 after retry RPC@1314701861) -> (continuation of Deferred@830033767 after retry RPC@421326471) -> (continuation of Deferred@1605791663 after retry RPC@555797000) -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@33a6a791 -> wakeup thread Time-limited test -> (continuation of Deferred@1507183059 after retry RPC@1598650199) -> (continuation of Deferred@1522263530 after retry RPC@1314701861) -> (continuation of Deferred@830033767 after retry RPC@421326471) -> (continuation of Deferred@1605791663 after retry RPC@555797000) -> (continuation of Deferred@1841406144 after retry RPC@1251362810) -> (continuation of Deferred@1715920893 after retry RPC@513273176))) after lookup
19:59:16.216 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:16.217 [DEBUG - kudu-nio-0] (Connection.java:757) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] Successfully connected to peer
19:59:16.233 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x176f99dd, L:/127.0.0.1:41866 - R:/127.9.144.2:39377] using TOKEN/null
19:59:16.276 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.144.3:43859
19:59:16.276 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

19:59:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.277595  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data
19:59:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/logs
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.3:43859
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.3
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37285
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.3
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:59:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:16.537724 11311 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111) [suppressed 6 similar messages]
19:59:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:16.595384 11653 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:59:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:16.595677 11653 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:16.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:16.595793 11653 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:59:16.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:16.603967 11653 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:59:16.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:16.604159 11653 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.3
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.613991 11653 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.3:43859
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:59:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.3
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37285
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.3
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/logs
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:59:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:16.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.616063 11653 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:16.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.617954 11653 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:59:16.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:16.629365 11661 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:16.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:16.629729 11660 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:16.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:16.630960 11663 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:16.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.631251 11653 server_base.cc:1061] running on GCE node
19:59:16.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.631824 11653 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:16.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.632704 11653 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:16.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.633888 11653 hybrid_clock.cc:648] HybridClock initialized: now 1783540756633822 us; error 57 us; skew 500 ppm
19:59:16.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.634157 11653 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.636065 11653 webserver.cc:533] Webserver started at http://127.9.144.3:37285/ using document root <none> and password file <none>
19:59:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.636714 11653 fs_manager.cc:362] Metadata directory not provided
19:59:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.636794 11653 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:16.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.642005 11653 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.001s	sys 0.003s
19:59:16.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.644579 11669 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:59:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.645993 11653 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
19:59:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.646116 11653 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "218dc41941a04327b46d9ca11df80df0"
19:59:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:59:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.646821 11653 fs_report.cc:389] FS layout report
19:59:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/data
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:16.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.669037 11653 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:59:16.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.669765 11653 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.669992 11653 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.670814 11653 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:59:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.672921 11676 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
19:59:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.680289 11653 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
19:59:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.680382 11653 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
19:59:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.680507 11653 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
19:59:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.683473 11653 ts_tablet_manager.cc:616] Registered 1 tablets
19:59:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.683568 11653 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.001s
19:59:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.683887 11676 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap starting.
19:59:16.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.709062 11676 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Log is configured to *not* fsync() on all Append() calls
19:59:16.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.722951 11653 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.3:43859
19:59:16.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.722999 11783 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.3:43859 every 8 connection(s)
19:59:16.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.724299 11653 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:59:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.729192  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 11653
19:59:16.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:16.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.743359 11676 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:59:16.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.744024 11676 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap complete.
19:59:16.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.745379 11676 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Time spent bootstrapping tablet: real 0.062s	user 0.048s	sys 0.011s
19:59:16.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.752177 11676 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:16.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.753122 11676 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 218dc41941a04327b46d9ca11df80df0, State: Initialized, Role: FOLLOWER
19:59:16.754 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:59:16.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.753991 11676 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:16.755 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.144.2:39377
19:59:16.755 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

19:59:16.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.756055  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 11295
19:59:16.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.774577 11786 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:59:16.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.775009 11786 heartbeater.cc:461] Registering TS with master...
19:59:16.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.775286 11676 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Time spent starting tablet: real 0.030s	user 0.016s	sys 0.001s
19:59:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.776315 11786 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:59:16.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.779361  9904 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:59:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.784016 11419 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.785743 11784 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:59:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.785887 11784 heartbeater.cc:461] Registering TS with master...
19:59:16.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.786429 11784 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:59:16.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.788030  9834 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:59:16.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.789479  9834 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.3:58081
19:59:16.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.791898 11784 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:59:16.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.794945 11785 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:59:16.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.795073 11785 heartbeater.cc:461] Registering TS with master...
19:59:16.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.795361 11785 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:59:16.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.796656  9975 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:59:16.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:16.926029  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 11295
19:59:16.940 [DEBUG - kudu-nio-0] (Connection.java:239) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] handling channelInactive
19:59:16.940 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:16.940 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state READY due to: connection closed
19:59:16.940 [DEBUG - Time-limited test] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:16.941 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:16.941 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:16.941 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:16.953 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:16.954 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:16.954 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [1ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [13ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1290798368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5ce4723c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@662aa0fc -> wakeup thread Time-limited test)) after lookup
19:59:16.954 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:16.956 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:16.956 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:16.956 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:16.973 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:16.973 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:16.974 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [1ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [13ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [16ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [16ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [33ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1290798368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5ce4723c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@662aa0fc -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284))) after lookup
19:59:16.974 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:16.975 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:16.976 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:16.976 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.013 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:17.013 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:17.014 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [1ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [13ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [16ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [16ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [33ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [34ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [36ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [36ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [73ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1290798368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5ce4723c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284) -> (continuation of Deferred@2064805427 after retry RPC@1606282945), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@662aa0fc -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284) -> (continuation of Deferred@2064805427 after retry RPC@1606282945))) after lookup
19:59:17.014 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:17.016 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.016 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.016 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.133 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:17.134 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:17.134 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=194), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [1ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [13ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [16ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [16ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [33ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [34ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [36ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [36ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [73ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [74ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [76ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [76ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [190ms] refreshing cache from master, [190ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [193ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1290798368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5ce4723c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284) -> (continuation of Deferred@2064805427 after retry RPC@1606282945) -> (continuation of Deferred@452683905 after retry RPC@704661241), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@662aa0fc -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284) -> (continuation of Deferred@2064805427 after retry RPC@1606282945) -> (continuation of Deferred@452683905 after retry RPC@704661241))) after lookup
19:59:17.135 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:17.136 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.136 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.136 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.233 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:17.233 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:17.234 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=293), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [1ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [13ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [16ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [16ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [33ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [34ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [36ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [36ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [73ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [74ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [76ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [76ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [190ms] refreshing cache from master, [190ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [193ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [195ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [196ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [196ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [290ms] refreshing cache from master, [291ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [293ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1290798368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5ce4723c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284) -> (continuation of Deferred@2064805427 after retry RPC@1606282945) -> (continuation of Deferred@452683905 after retry RPC@704661241) -> (continuation of Deferred@1439962353 after retry RPC@2110502716), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@662aa0fc -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284) -> (continuation of Deferred@2064805427 after retry RPC@1606282945) -> (continuation of Deferred@452683905 after retry RPC@704661241) -> (continuation of Deferred@1439962353 after retry RPC@2110502716))) after lookup
19:59:17.235 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:17.236 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.236 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.236 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.833 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:17.833 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)[L]]}]
19:59:17.834 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=893), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [1ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [13ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [16ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [16ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [33ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [34ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [36ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [36ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [73ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [74ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [76ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [76ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [190ms] refreshing cache from master, [190ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [193ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [195ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [196ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [196ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [290ms] refreshing cache from master, [291ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [293ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [294ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [296ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [296ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [890ms] refreshing cache from master, [890ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [892ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1290798368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5ce4723c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284) -> (continuation of Deferred@2064805427 after retry RPC@1606282945) -> (continuation of Deferred@452683905 after retry RPC@704661241) -> (continuation of Deferred@1439962353 after retry RPC@2110502716) -> (continuation of Deferred@334971292 after retry RPC@858089097), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@662aa0fc -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284) -> (continuation of Deferred@2064805427 after retry RPC@1606282945) -> (continuation of Deferred@452683905 after retry RPC@704661241) -> (continuation of Deferred@1439962353 after retry RPC@2110502716) -> (continuation of Deferred@334971292 after retry RPC@858089097))) after lookup
19:59:17.835 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] connecting to peer
19:59:17.836 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.836 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] cleaning up while in state CONNECTING due to: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:17.836 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377
19:59:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.052821 11798 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:59:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.053105 11798 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.054641 11798 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:18.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:18.056185 11673 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:59:18.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:18.057880 11673 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:59:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.061429 11555 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74" is_pre_election: true
19:59:18.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.062307 11673 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0; no voters: 2d2fa818193a44629aea25a754e9ec74, d387b01aa6714e8893e35a67daefebb8
19:59:18.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.062669 11798 raft_consensus.cc:2749] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
19:59:18.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.423777 11803 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader d387b01aa6714e8893e35a67daefebb8)
19:59:18.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.423990 11803 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:18.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.425427 11803 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:18.427340 11490 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:59:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:18.428933 11490 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:59:18.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.433379 11738 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "2d2fa818193a44629aea25a754e9ec74" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "218dc41941a04327b46d9ca11df80df0" is_pre_election: true
19:59:18.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.433702 11738 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2d2fa818193a44629aea25a754e9ec74 in term 7.
19:59:18.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.434533 11488 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, 2d2fa818193a44629aea25a754e9ec74; no voters: d387b01aa6714e8893e35a67daefebb8
19:59:18.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.434875 11803 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 8 FOLLOWER]: Leader pre-election won for term 9
19:59:18.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.434981 11803 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 8 FOLLOWER]: Starting leader election (detected failure of leader d387b01aa6714e8893e35a67daefebb8)
19:59:18.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.435045 11803 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 8 FOLLOWER]: Advancing to term 9
19:59:18.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.436928 11803 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:18.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.437599 11803 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 9 election: Requested vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:18.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.438055 11738 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "2d2fa818193a44629aea25a754e9ec74" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "218dc41941a04327b46d9ca11df80df0"
19:59:18.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.438238 11738 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 7 FOLLOWER]: Advancing to term 9
19:59:18.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:18.438905 11490 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111)
19:59:18.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.442616 11738 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2d2fa818193a44629aea25a754e9ec74 in term 9.
19:59:18.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.442991 11488 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, 2d2fa818193a44629aea25a754e9ec74; no voters: d387b01aa6714e8893e35a67daefebb8
19:59:18.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.443289 11803 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 9 FOLLOWER]: Leader election won for term 9
19:59:18.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.443691 11803 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 9 LEADER]: Becoming Leader. State: Replica: 2d2fa818193a44629aea25a754e9ec74, State: Running, Role: LEADER
19:59:18.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.444156 11803 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:18.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.447278  9834 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 reported cstate change: term changed from 8 to 9, leader changed from d387b01aa6714e8893e35a67daefebb8 (127.9.144.2) to 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1). New cstate: current_term: 9 leader_uuid: "2d2fa818193a44629aea25a754e9ec74" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: HEALTHY } } }
19:59:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:18.835541 11490 consensus_peers.cc:597] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 -> Peer d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377): Couldn't send request to peer d387b01aa6714e8893e35a67daefebb8. Status: Network error: Client connection negotiation failed: client connection to 127.9.144.2:39377: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:59:18.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.971091 11738 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 9 FOLLOWER]: Refusing update from remote peer 2d2fa818193a44629aea25a754e9ec74: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
19:59:18.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:18.971850 11803 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [LEADER]: Connected to new peer: Peer: permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
19:59:20.793 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:20.793 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:20.794 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=3853), Traces: [0ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [1ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [2ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [13ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [14ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [16ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [16ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [33ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [34ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [36ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [36ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [73ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [74ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [76ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [76ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [190ms] refreshing cache from master, [190ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [193ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [195ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [196ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [196ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [290ms] refreshing cache from master, [291ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [293ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [294ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [296ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [296ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [890ms] refreshing cache from master, [890ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [892ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [894ms] sending RPC to server d387b01aa6714e8893e35a67daefebb8, [896ms] received response from server d387b01aa6714e8893e35a67daefebb8: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [896ms] delaying RPC due to: Network error: Failed to connect to peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377): Connection refused: /127.9.144.2:39377, [3850ms] refreshing cache from master, [3850ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [3852ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1290798368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@5ce4723c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284) -> (continuation of Deferred@2064805427 after retry RPC@1606282945) -> (continuation of Deferred@452683905 after retry RPC@704661241) -> (continuation of Deferred@1439962353 after retry RPC@2110502716) -> (continuation of Deferred@334971292 after retry RPC@858089097) -> (continuation of Deferred@2024734508 after retry RPC@1613221790), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@662aa0fc -> wakeup thread Time-limited test -> (continuation of Deferred@1312116846 after retry RPC@2099979284) -> (continuation of Deferred@2064805427 after retry RPC@1606282945) -> (continuation of Deferred@452683905 after retry RPC@704661241) -> (continuation of Deferred@1439962353 after retry RPC@2110502716) -> (continuation of Deferred@334971292 after retry RPC@858089097) -> (continuation of Deferred@2024734508 after retry RPC@1613221790))) after lookup
19:59:20.795 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:20.795 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] Successfully connected to peer
19:59:20.811 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xbfaec27b, L:/127.0.0.1:57068 - R:/127.9.144.1:34195] using TOKEN/null
19:59:20.831 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.144.2:39377
19:59:20.831 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

19:59:20.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:20.832331  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:20.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:20.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:59:20.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data
19:59:20.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:20.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:20.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/logs
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.2:39377
19:59:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.2
19:59:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36739
19:59:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.2
19:59:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:59:21.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:21.146978 11841 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:59:21.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:21.147260 11841 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:21.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:21.147338 11841 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:59:21.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:21.155573 11841 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:59:21.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:21.155736 11841 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.2
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.165380 11841 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.2:39377
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.2
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36739
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.2
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/logs
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:59:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:21.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.166895 11841 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.168598 11841 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:59:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:21.177413 11848 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:21.177384 11849 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:21.178484 11851 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.179010 11841 server_base.cc:1061] running on GCE node
19:59:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.179580 11841 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.180477 11841 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.181677 11841 hybrid_clock.cc:648] HybridClock initialized: now 1783540761181630 us; error 32 us; skew 500 ppm
19:59:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.182046 11841 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:21.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.184137 11841 webserver.cc:533] Webserver started at http://127.9.144.2:36739/ using document root <none> and password file <none>
19:59:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.184880 11841 fs_manager.cc:362] Metadata directory not provided
19:59:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.184980 11841 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:21.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.190359 11841 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.001s	sys 0.003s
19:59:21.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.193105 11857 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:59:21.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.194362 11841 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.194512 11841 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d387b01aa6714e8893e35a67daefebb8"
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.195240 11841 fs_report.cc:389] FS layout report
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/wal
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/data
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:21.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:21.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:21.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.219411 11841 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:59:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.220319 11841 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.220624 11841 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:21.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.221700 11841 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:59:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.223796 11864 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
19:59:21.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.229795 11841 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
19:59:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.229895 11841 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
19:59:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.230019 11841 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
19:59:21.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.233278 11841 ts_tablet_manager.cc:616] Registered 1 tablets
19:59:21.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.233392 11841 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
19:59:21.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.233825 11864 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Bootstrap starting.
19:59:21.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.262784 11864 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Log is configured to *not* fsync() on all Append() calls
19:59:21.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.278988 11841 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.2:39377
19:59:21.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.279078 11971 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.2:39377 every 8 connection(s)
19:59:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.280478 11841 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-1/data/info.pb
19:59:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.287221  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 11841
19:59:21.287 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:21.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.303676 11864 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:59:21.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.304378 11864 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Bootstrap complete.
19:59:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.306582 11864 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Time spent bootstrapping tablet: real 0.073s	user 0.052s	sys 0.018s
19:59:21.312 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:59:21.314 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.144.1:34195
19:59:21.314 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

19:59:21.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.314669  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 11472
19:59:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.325572 11864 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.326649 11864 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: d387b01aa6714e8893e35a67daefebb8, State: Initialized, Role: FOLLOWER
19:59:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.327884 11864 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.336844 11973 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:59:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.337272 11973 heartbeater.cc:461] Registering TS with master...
19:59:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.338565 11973 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:59:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.341143 11972 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:59:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.341279 11972 heartbeater.cc:461] Registering TS with master...
19:59:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.341843 11972 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:59:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.343288  9834 ts_manager.cc:194] Re-registered known tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.343397  9975 ts_manager.cc:194] Re-registered known tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.344913  9834 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.2:40939
19:59:21.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.367218 11972 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:59:21.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.367924 11864 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8: Time spent starting tablet: real 0.061s	user 0.014s	sys 0.027s
19:59:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.369752 11977 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:59:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.369910 11977 heartbeater.cc:461] Registering TS with master...
19:59:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.370203 11977 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:59:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.371357 11596 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.371558  9904 ts_manager.cc:194] Re-registered known tserver with Master: d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:21.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:21.472064  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 11472
19:59:21.487 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:21.487 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] handling channelInactive
19:59:21.487 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state READY due to: connection closed
19:59:21.488 [DEBUG - Time-limited test] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:21.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.489 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.493 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:21.493 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:21.493 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=6), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [3ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [6ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@746859634(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@6ad89bf -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@23987015 -> wakeup thread Time-limited test)) after lookup
19:59:21.494 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:21.495 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.495 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.495 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.513 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:21.513 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:21.513 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [3ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [6ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [7ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [8ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [25ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@746859634(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@6ad89bf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1810554469 after retry RPC@185734101), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@23987015 -> wakeup thread Time-limited test -> (continuation of Deferred@1810554469 after retry RPC@185734101))) after lookup
19:59:21.514 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:21.515 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.515 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.515 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.554 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:21.554 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:21.559 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [3ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [6ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [7ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [8ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [25ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [26ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [28ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [28ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [63ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [66ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@746859634(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@6ad89bf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1810554469 after retry RPC@185734101) -> (continuation of Deferred@1766975402 after retry RPC@1925635566), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@23987015 -> wakeup thread Time-limited test -> (continuation of Deferred@1810554469 after retry RPC@185734101) -> (continuation of Deferred@1766975402 after retry RPC@1925635566))) after lookup
19:59:21.560 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:21.561 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.561 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.562 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.593 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:21.593 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:21.593 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=106), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [3ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [6ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [7ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [8ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [25ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [26ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [28ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [28ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [63ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [66ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [73ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [74ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [75ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [106ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@746859634(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@6ad89bf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1810554469 after retry RPC@185734101) -> (continuation of Deferred@1766975402 after retry RPC@1925635566) -> (continuation of Deferred@1755516220 after retry RPC@751735915), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@23987015 -> wakeup thread Time-limited test -> (continuation of Deferred@1810554469 after retry RPC@185734101) -> (continuation of Deferred@1766975402 after retry RPC@1925635566) -> (continuation of Deferred@1755516220 after retry RPC@751735915))) after lookup
19:59:21.594 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:21.595 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.595 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.595 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.753 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:21.754 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:21.754 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=266), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [3ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [6ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [7ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [8ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [25ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [26ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [28ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [28ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [63ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [66ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [73ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [74ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [75ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [106ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [107ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [108ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [108ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [263ms] refreshing cache from master, [264ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [266ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@746859634(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@6ad89bf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1810554469 after retry RPC@185734101) -> (continuation of Deferred@1766975402 after retry RPC@1925635566) -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@23987015 -> wakeup thread Time-limited test -> (continuation of Deferred@1810554469 after retry RPC@185734101) -> (continuation of Deferred@1766975402 after retry RPC@1925635566) -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483))) after lookup
19:59:21.754 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:21.755 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.756 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:21.756 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195
19:59:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.763661 11986 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 2d2fa818193a44629aea25a754e9ec74)
19:59:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.763873 11986 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.764658 11986 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:22.769347 11673 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:22.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.772481 11926 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8" is_pre_election: true
19:59:22.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.772902 11926 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 218dc41941a04327b46d9ca11df80df0 in term 8.
19:59:22.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.773759 11673 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, d387b01aa6714e8893e35a67daefebb8; no voters: 2d2fa818193a44629aea25a754e9ec74
19:59:22.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.774031 11986 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 9 FOLLOWER]: Leader pre-election won for term 10
19:59:22.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.774147 11986 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 2d2fa818193a44629aea25a754e9ec74)
19:59:22.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.774242 11986 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 9 FOLLOWER]: Advancing to term 10
19:59:22.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.776186 11986 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:22.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.776803 11986 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 10 election: Requested vote from peers d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.777200 11926 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "218dc41941a04327b46d9ca11df80df0" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8"
19:59:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.777379 11926 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 8 FOLLOWER]: Advancing to term 10
19:59:22.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:22.778108 11673 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111)
19:59:22.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.781216 11926 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 218dc41941a04327b46d9ca11df80df0 in term 10.
19:59:22.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.781669 11673 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 218dc41941a04327b46d9ca11df80df0, d387b01aa6714e8893e35a67daefebb8; no voters: 2d2fa818193a44629aea25a754e9ec74
19:59:22.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.781921 11986 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 10 FOLLOWER]: Leader election won for term 10
19:59:22.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.782305 11986 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 10 LEADER]: Becoming Leader. State: Replica: 218dc41941a04327b46d9ca11df80df0, State: Running, Role: LEADER
19:59:22.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.782771 11986 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:22.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.786238  9834 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 reported cstate change: term changed from 9 to 10, leader changed from 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1) to 218dc41941a04327b46d9ca11df80df0 (127.9.144.3). New cstate: current_term: 10 leader_uuid: "218dc41941a04327b46d9ca11df80df0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: UNKNOWN } } }
19:59:22.893 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:22.893 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:22.894 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1406), Traces: [0ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [2ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [3ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [6ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [7ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [8ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [25ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [26ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [28ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [28ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [63ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [66ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [73ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [74ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [75ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [106ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [107ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [108ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [108ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [263ms] refreshing cache from master, [264ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [266ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [267ms] sending RPC to server 2d2fa818193a44629aea25a754e9ec74, [269ms] received response from server 2d2fa818193a44629aea25a754e9ec74: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [269ms] delaying RPC due to: Network error: Failed to connect to peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195): Connection refused: /127.9.144.1:34195, [1403ms] refreshing cache from master, [1403ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [1406ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@746859634(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@6ad89bf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1810554469 after retry RPC@185734101) -> (continuation of Deferred@1766975402 after retry RPC@1925635566) -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483) -> (continuation of Deferred@734728762 after retry RPC@414578541), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@23987015 -> wakeup thread Time-limited test -> (continuation of Deferred@1810554469 after retry RPC@185734101) -> (continuation of Deferred@1766975402 after retry RPC@1925635566) -> (continuation of Deferred@1755516220 after retry RPC@751735915) -> (continuation of Deferred@1179651198 after retry RPC@518366483) -> (continuation of Deferred@734728762 after retry RPC@414578541))) after lookup
19:59:22.895 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:22.896 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] Successfully connected to peer
19:59:22.920 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0x48e5d5bc, L:/127.0.0.1:39764 - R:/127.9.144.3:43859] using TOKEN/null
19:59:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.926476 11926 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 10 FOLLOWER]: Refusing update from remote peer 218dc41941a04327b46d9ca11df80df0: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 38. (index mismatch)
19:59:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:22.926548 11673 consensus_peers.cc:597] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 -> Peer 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195): Couldn't send request to peer 2d2fa818193a44629aea25a754e9ec74. Status: Network error: Client connection negotiation failed: client connection to 127.9.144.1:34195: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:59:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.927340 11986 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
19:59:22.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.943864 12005 mvcc.cc:204] Tried to move back new op lower bound from 7305382964935987200 to 7305382964361437184. Current Snapshot: MvccSnapshot[applied={T|T < 7305382964935987200}]
19:59:22.962 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.144.1:34195
19:59:22.963 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

19:59:22.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:22.963663  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:22.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/logs
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.1:34195
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.1
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40347
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.1
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:22.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:22.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:59:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:23.277858 12013 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:59:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:23.278143 12013 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:23.278223 12013 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:59:23.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:23.286734 12013 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:59:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:23.286885 12013 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.1
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.297173 12013 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.1:34195
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.1
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40347
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.1
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/logs
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.298796 12013 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:23.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.300544 12013 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:59:23.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:23.309237 12018 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:23.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:23.309286 12019 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:23.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:23.310210 12021 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:23.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.310535 12013 server_base.cc:1061] running on GCE node
19:59:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.311087 12013 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:23.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.312003 12013 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:23.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.313181 12013 hybrid_clock.cc:648] HybridClock initialized: now 1783540763313118 us; error 59 us; skew 500 ppm
19:59:23.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.313462 12013 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:23.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.315366 12013 webserver.cc:533] Webserver started at http://127.9.144.1:40347/ using document root <none> and password file <none>
19:59:23.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.316032 12013 fs_manager.cc:362] Metadata directory not provided
19:59:23.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.316114 12013 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:23.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.321230 12013 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.003s	sys 0.000s
19:59:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.323740 12027 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:59:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.324972 12013 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
19:59:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.325134 12013 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d2fa818193a44629aea25a754e9ec74"
19:59:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:49 on dist-test-slave-trb6"
19:59:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.325754 12013 fs_report.cc:389] FS layout report
19:59:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/wal
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/data
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:23.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.339299 12013 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:59:23.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.340003 12013 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:23.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.340222 12013 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:23.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.341130 12013 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:59:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.342933 12034 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
19:59:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.348583 12013 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
19:59:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.348664 12013 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
19:59:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.348758 12013 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
19:59:23.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.351821 12013 ts_tablet_manager.cc:616] Registered 1 tablets
19:59:23.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.351907 12013 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
19:59:23.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.352519 12034 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Bootstrap starting.
19:59:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.378129 12034 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Log is configured to *not* fsync() on all Append() calls
19:59:23.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.391539 12013 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.1:34195
19:59:23.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.391553 12141 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.1:34195 every 8 connection(s)
19:59:23.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.393236 12013 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-0/data/info.pb
19:59:23.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.398543  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 12013
19:59:23.431 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:59:23.435 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.9.144.3:43859
19:59:23.435 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

19:59:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.435779  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 11653
19:59:23.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.438228 12034 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:59:23.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.438967 12034 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Bootstrap complete.
19:59:23.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.440511 12034 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Time spent bootstrapping tablet: real 0.089s	user 0.051s	sys 0.017s
19:59:23.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.445569 12142 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:59:23.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.445947 12142 heartbeater.cc:461] Registering TS with master...
19:59:23.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.447197 12142 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:59:23.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.447870 12034 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:23.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.448976 12034 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2d2fa818193a44629aea25a754e9ec74, State: Initialized, Role: FOLLOWER
19:59:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.449931 12034 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.451401  9834 ts_manager.cc:194] Re-registered known tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:23.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.452975  9834 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.1:36295
19:59:23.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.463128 12143 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:59:23.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.463321 12143 heartbeater.cc:461] Registering TS with master...
19:59:23.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.463733 12143 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:59:23.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.465181  9975 ts_manager.cc:194] Re-registered known tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:23.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.468969 12144 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:59:23.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.469148 12144 heartbeater.cc:461] Registering TS with master...
19:59:23.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.469476 12144 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:59:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.470834  9904 ts_manager.cc:194] Re-registered known tserver with Master: 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:23.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.472523 12142 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:59:23.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.473493 12034 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74: Time spent starting tablet: real 0.033s	user 0.015s	sys 0.016s
19:59:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.479516 11779 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.482302 12096 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 9 FOLLOWER]: Advancing to term 10
19:59:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:23.582933  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 11653
19:59:23.596 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] handling channelInactive
19:59:23.596 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:23.597 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state READY due to: connection closed
19:59:23.597 [DEBUG - Time-limited test] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:23.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.599 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.613 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:23.613 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:23.613 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [2ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [2ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1015640183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@398bd508 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3926ddb -> wakeup thread Time-limited test)) after lookup
19:59:23.614 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:23.615 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.615 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.615 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.633 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:23.633 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:23.634 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [2ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [2ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [18ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [18ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [36ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1015640183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@398bd508 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3926ddb -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284))) after lookup
19:59:23.634 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:23.635 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.635 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.635 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.652 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:23.653 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:23.653 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [2ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [2ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [18ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [18ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [36ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [37ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [38ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [38ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1015640183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@398bd508 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284) -> (continuation of Deferred@1051898125 after retry RPC@698244917), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3926ddb -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284) -> (continuation of Deferred@1051898125 after retry RPC@698244917))) after lookup
19:59:23.654 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:23.655 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.655 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.655 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.694 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:23.694 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:23.695 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=97), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [2ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [2ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [18ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [18ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [36ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [37ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [38ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [38ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [56ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [58ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [58ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [97ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1015640183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@398bd508 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284) -> (continuation of Deferred@1051898125 after retry RPC@698244917) -> (continuation of Deferred@1183662099 after retry RPC@498194432), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3926ddb -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284) -> (continuation of Deferred@1051898125 after retry RPC@698244917) -> (continuation of Deferred@1183662099 after retry RPC@498194432))) after lookup
19:59:23.695 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:23.696 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.696 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.697 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.713 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:23.713 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:23.714 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=116), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [2ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [2ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [18ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [18ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [36ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [37ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [38ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [38ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [56ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [58ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [58ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [97ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [98ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [100ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [100ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [116ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1015640183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@398bd508 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284) -> (continuation of Deferred@1051898125 after retry RPC@698244917) -> (continuation of Deferred@1183662099 after retry RPC@498194432) -> (continuation of Deferred@915979763 after retry RPC@628753144), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3926ddb -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284) -> (continuation of Deferred@1051898125 after retry RPC@698244917) -> (continuation of Deferred@1183662099 after retry RPC@498194432) -> (continuation of Deferred@915979763 after retry RPC@628753144))) after lookup
19:59:23.715 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:23.716 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.716 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:23.716 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:24.814 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:24.814 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)[L],2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195),d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:24.814 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1217), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [2ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [2ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [18ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [18ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [36ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [37ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [38ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [38ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [56ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [58ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [58ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [97ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [98ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [100ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [100ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [116ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [117ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [119ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [119ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1213ms] refreshing cache from master, [1214ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [1216ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1015640183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@398bd508 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284) -> (continuation of Deferred@1051898125 after retry RPC@698244917) -> (continuation of Deferred@1183662099 after retry RPC@498194432) -> (continuation of Deferred@915979763 after retry RPC@628753144) -> (continuation of Deferred@1639402349 after retry RPC@338688097), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3926ddb -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284) -> (continuation of Deferred@1051898125 after retry RPC@698244917) -> (continuation of Deferred@1183662099 after retry RPC@498194432) -> (continuation of Deferred@915979763 after retry RPC@628753144) -> (continuation of Deferred@1639402349 after retry RPC@338688097))) after lookup
19:59:24.815 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] connecting to peer
19:59:24.816 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:24.816 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] cleaning up while in state CONNECTING due to: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:24.816 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859) for tablet 07e9b26c7a2f46edb82f05d9afd8005e: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859
19:59:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.856407 12163 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 218dc41941a04327b46d9ca11df80df0)
19:59:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.856601 12163 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.858080 12163 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1:34195)
19:59:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:24.859503 11859 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:24.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:24.861011 11859 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:24.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.864566 12096 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "d387b01aa6714e8893e35a67daefebb8" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "2d2fa818193a44629aea25a754e9ec74" is_pre_election: true
19:59:24.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.865471 11861 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0, 2d2fa818193a44629aea25a754e9ec74
19:59:24.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.865803 12163 raft_consensus.cc:2749] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 10 FOLLOWER]: Leader pre-election lost for term 11. Reason: could not achieve majority
19:59:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.989331 12168 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 218dc41941a04327b46d9ca11df80df0)
19:59:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.989560 12168 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:24.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.990991 12168 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:24.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:24.992444 12029 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:24.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:24.994069 12029 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.997334 11926 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "2d2fa818193a44629aea25a754e9ec74" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8" is_pre_election: true
19:59:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.997563 11926 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2d2fa818193a44629aea25a754e9ec74 in term 10.
19:59:24.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.998075 12031 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2d2fa818193a44629aea25a754e9ec74, d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0
19:59:24.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.998442 12168 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 10 FOLLOWER]: Leader pre-election won for term 11
19:59:24.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.998562 12168 raft_consensus.cc:493] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 218dc41941a04327b46d9ca11df80df0)
19:59:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:24.998657 12168 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 10 FOLLOWER]: Advancing to term 11
19:59:25.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.000681 12168 raft_consensus.cc:515] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:25.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.001291 12168 leader_election.cc:290] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 11 election: Requested vote from peers 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859), d387b01aa6714e8893e35a67daefebb8 (127.9.144.2:39377)
19:59:25.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.001796 11926 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "07e9b26c7a2f46edb82f05d9afd8005e" candidate_uuid: "2d2fa818193a44629aea25a754e9ec74" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "d387b01aa6714e8893e35a67daefebb8"
19:59:25.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.001991 11926 raft_consensus.cc:3060] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 10 FOLLOWER]: Advancing to term 11
19:59:25.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.002601 12029 leader_election.cc:336] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111)
19:59:25.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.003880 11926 raft_consensus.cc:2468] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2d2fa818193a44629aea25a754e9ec74 in term 11.
19:59:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.004225 12031 leader_election.cc:304] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2d2fa818193a44629aea25a754e9ec74, d387b01aa6714e8893e35a67daefebb8; no voters: 218dc41941a04327b46d9ca11df80df0
19:59:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.004479 12168 raft_consensus.cc:2804] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 11 FOLLOWER]: Leader election won for term 11
19:59:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.004837 12168 raft_consensus.cc:697] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [term 11 LEADER]: Becoming Leader. State: Replica: 2d2fa818193a44629aea25a754e9ec74, State: Running, Role: LEADER
19:59:25.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.005311 12168 consensus_queue.cc:237] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.008507  9834 catalog_manager.cc:5697] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 reported cstate change: term changed from 10 to 11, leader changed from 218dc41941a04327b46d9ca11df80df0 (127.9.144.3) to 2d2fa818193a44629aea25a754e9ec74 (127.9.144.1). New cstate: current_term: 11 leader_uuid: "2d2fa818193a44629aea25a754e9ec74" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } health_report { overall_health: HEALTHY } } }
19:59:25.253 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 07e9b26c7a2f46edb82f05d9afd8005e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358' with partition [<start>, <end>)
19:59:25.253 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=07e9b26c7a2f46edb82f05d9afd8005e@[218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859),2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)[L],d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)]}]
19:59:25.254 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=07e9b26c7a2f46edb82f05d9afd8005e, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1656), Traces: [0ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [2ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [2ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [15ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [16ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [18ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [18ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [36ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [37ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [38ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [38ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [55ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [56ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [58ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [58ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [97ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [98ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [100ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [100ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [116ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [117ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [119ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [119ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1213ms] refreshing cache from master, [1214ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [1216ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, [1218ms] sending RPC to server 218dc41941a04327b46d9ca11df80df0, [1219ms] received response from server 218dc41941a04327b46d9ca11df80df0: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1219ms] delaying RPC due to: Network error: Failed to connect to peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859): Connection refused: /127.9.144.3:43859, [1653ms] refreshing cache from master, [1653ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.144.62:44017, [1656ms] Sub RPC GetTableLocations: received response from server master-127.9.144.62:44017: OK, deferred=Deferred@1015640183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$241/0x00007ff2a0338f98@398bd508 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284) -> (continuation of Deferred@1051898125 after retry RPC@698244917) -> (continuation of Deferred@1183662099 after retry RPC@498194432) -> (continuation of Deferred@915979763 after retry RPC@628753144) -> (continuation of Deferred@1639402349 after retry RPC@338688097) -> (continuation of Deferred@621368947 after retry RPC@1317953091), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3926ddb -> wakeup thread Time-limited test -> (continuation of Deferred@1380380938 after retry RPC@1110090284) -> (continuation of Deferred@1051898125 after retry RPC@698244917) -> (continuation of Deferred@1183662099 after retry RPC@498194432) -> (continuation of Deferred@915979763 after retry RPC@628753144) -> (continuation of Deferred@1639402349 after retry RPC@338688097) -> (continuation of Deferred@621368947 after retry RPC@1317953091))) after lookup
19:59:25.255 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] connecting to peer
19:59:25.256 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] Successfully connected to peer
19:59:25.274 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0x7b820afc, L:/127.0.0.1:37796 - R:/127.9.144.1:34195] using TOKEN/null
19:59:25.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.279386 11926 raft_consensus.cc:1275] T 07e9b26c7a2f46edb82f05d9afd8005e P d387b01aa6714e8893e35a67daefebb8 [term 11 FOLLOWER]: Refusing update from remote peer 2d2fa818193a44629aea25a754e9ec74: Log matching property violated. Preceding OpId in replica: term: 10 index: 40. Preceding OpId from leader: term: 11 index: 42. (index mismatch)
19:59:25.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.280121 12173 consensus_queue.cc:1048] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 40, Time since last communication: 0.000s
19:59:25.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.280372 12029 consensus_peers.cc:597] T 07e9b26c7a2f46edb82f05d9afd8005e P 2d2fa818193a44629aea25a754e9ec74 -> Peer 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859): Couldn't send request to peer 218dc41941a04327b46d9ca11df80df0. Status: Network error: Client connection negotiation failed: client connection to 127.9.144.3:43859: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:59:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.289973 12186 mvcc.cc:204] Tried to move back new op lower bound from 7305382974577598464 to 7305382973464924160. Current Snapshot: MvccSnapshot[applied={T|T < 7305382974577598464}]
19:59:25.307 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.9.144.3:43859
19:59:25.307 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.307988  9792 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/logs
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.3:43859
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.3
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37285
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.3
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:59:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.623126 12192 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
19:59:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.623413 12192 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.623495 12192 flags.cc:432] Enabled unsafe flag: --never_fsync=true
19:59:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.631690 12192 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
19:59:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.631855 12192 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.144.3
19:59:25.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.641573 12192 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:25.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:25.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.144.20:35497
19:59:25.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:25.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:25.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data
19:59:25.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.144.3:43859
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.144.3
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37285
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.144.62:44017,127.9.144.61:40553,127.9.144.60:37879
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.144.3
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/logs
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 19:43:25 UTC on bdcb31816ec0
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12778
19:59:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.643543 12192 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:25.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.645859 12192 file_cache.cc:504] Constructed file cache file cache with capacity 419430
19:59:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.654950 12199 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.654954 12198 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.656034 12192 server_base.cc:1061] running on GCE node
19:59:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.656133 12201 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.656733 12192 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.657727 12192 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.658921 12192 hybrid_clock.cc:648] HybridClock initialized: now 1783540765658874 us; error 36 us; skew 500 ppm
19:59:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.659228 12192 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:25.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.661269 12192 webserver.cc:533] Webserver started at http://127.9.144.3:37285/ using document root <none> and password file <none>
19:59:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.661983 12192 fs_manager.cc:362] Metadata directory not provided
19:59:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.662081 12192 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:25.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.667312 12192 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.005s	sys 0.000s
19:59:25.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.669972 12207 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
19:59:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.671283 12192 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.671432 12192 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data,/tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "218dc41941a04327b46d9ca11df80df0"
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 19:58:50 on dist-test-slave-trb6"
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.672130 12192 fs_report.cc:389] FS layout report
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/wal
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/data
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.685992 12192 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
19:59:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.686790 12192 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.687081 12192 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.688117 12192 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
19:59:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.690184 12214 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
19:59:25.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.695720 12192 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
19:59:25.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.695808 12192 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
19:59:25.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.695947 12192 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
19:59:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.698963 12192 ts_tablet_manager.cc:616] Registered 1 tablets
19:59:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.699059 12192 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.001s	sys 0.000s
19:59:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.699440 12214 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap starting.
19:59:25.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.726261 12214 log.cc:826] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Log is configured to *not* fsync() on all Append() calls
19:59:25.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.736568 12192 rpc_server.cc:307] RPC server started. Bound to: 127.9.144.3:43859
19:59:25.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.736613 12321 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.144.3:43859 every 8 connection(s)
19:59:25.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.737923 12192 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskvUDI3f/test-tmp/mini-kudu-cluster3142084106935173165/ts-2/data/info.pb
19:59:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.741997  9792 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu as pid 12192
19:59:25.742 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
19:59:25.761 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:59:25.766 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783540731358, tablet=07e9b26c7a2f46edb82f05d9afd8005e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
19:59:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.770334 12214 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:59:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.770998 12214 tablet_bootstrap.cc:492] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Bootstrap complete.
19:59:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.772316 12214 ts_tablet_manager.cc:1403] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Time spent bootstrapping tablet: real 0.073s	user 0.052s	sys 0.017s
19:59:25.773 [DEBUG - main] (Connection.java:575) [peer master-127.9.144.62:44017(127.9.144.62:44017)] disconnecting while in state READY
19:59:25.774 [DEBUG - main] (Connection.java:575) [peer master-127.9.144.60:37879(127.9.144.60:37879)] disconnecting while in state READY
19:59:25.774 [DEBUG - main] (Connection.java:575) [peer d387b01aa6714e8893e35a67daefebb8(127.9.144.2:39377)] disconnecting while in state TERMINATED
19:59:25.775 [DEBUG - main] (Connection.java:575) [peer master-127.9.144.61:40553(127.9.144.61:40553)] disconnecting while in state READY
19:59:25.775 [DEBUG - main] (Connection.java:575) [peer 218dc41941a04327b46d9ca11df80df0(127.9.144.3:43859)] disconnecting while in state TERMINATED
19:59:25.775 [DEBUG - main] (Connection.java:575) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] disconnecting while in state READY
19:59:25.776 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.144.62:44017(127.9.144.62:44017)] handling channelInactive
19:59:25.776 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.144.62:44017(127.9.144.62:44017)] cleaning up while in state READY due to: connection closed
19:59:25.776 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.9.144.61:40553(127.9.144.61:40553)] handling channelInactive
19:59:25.777 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.9.144.61:40553(127.9.144.61:40553)] cleaning up while in state READY due to: connection closed
19:59:25.778 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.9.144.60:37879(127.9.144.60:37879)] handling channelInactive
19:59:25.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.144.60:37879(127.9.144.60:37879)] cleaning up while in state READY due to: connection closed
19:59:25.778 [DEBUG - kudu-nio-7] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:59:25.779 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] handling channelInactive
19:59:25.779 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2d2fa818193a44629aea25a754e9ec74(127.9.144.1:34195)] cleaning up while in state READY due to: connection closed
19:59:25.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.781344  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 12013
19:59:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.791036 12214 raft_consensus.cc:359] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.792012 12214 raft_consensus.cc:740] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 218dc41941a04327b46d9ca11df80df0, State: Initialized, Role: FOLLOWER
19:59:25.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.792827 12214 consensus_queue.cc:260] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "218dc41941a04327b46d9ca11df80df0" member_type: VOTER last_known_addr { host: "127.9.144.3" port: 43859 } } peers { permanent_uuid: "d387b01aa6714e8893e35a67daefebb8" member_type: VOTER last_known_addr { host: "127.9.144.2" port: 39377 } } peers { permanent_uuid: "2d2fa818193a44629aea25a754e9ec74" member_type: VOTER last_known_addr { host: "127.9.144.1" port: 34195 } }
19:59:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.826575 12323 heartbeater.cc:344] Connected to a master server at 127.9.144.60:37879
19:59:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.827279 12323 heartbeater.cc:461] Registering TS with master...
19:59:25.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.828486 12323 heartbeater.cc:507] Master 127.9.144.60:37879 requested a full tablet report, sending...
19:59:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.829241 12324 heartbeater.cc:344] Connected to a master server at 127.9.144.61:40553
19:59:25.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.829388 12324 heartbeater.cc:461] Registering TS with master...
19:59:25.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.829934 12324 heartbeater.cc:507] Master 127.9.144.61:40553 requested a full tablet report, sending...
19:59:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.832443  9975 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:59:25.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.834965  9904 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:59:25.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.837208 12137 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.839023 12214 ts_tablet_manager.cc:1434] T 07e9b26c7a2f46edb82f05d9afd8005e P 218dc41941a04327b46d9ca11df80df0: Time spent starting tablet: real 0.067s	user 0.037s	sys 0.018s
19:59:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.839213 12322 heartbeater.cc:344] Connected to a master server at 127.9.144.62:44017
19:59:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.839340 12322 heartbeater.cc:461] Registering TS with master...
19:59:25.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.839861 12322 heartbeater.cc:507] Master 127.9.144.62:44017 requested a full tablet report, sending...
19:59:25.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.841215  9834 ts_manager.cc:194] Re-registered known tserver with Master: 218dc41941a04327b46d9ca11df80df0 (127.9.144.3:43859)
19:59:25.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.842796  9834 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.9.144.3:52881
19:59:25.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.845600 12322 heartbeater.cc:499] Master 127.9.144.62:44017 was elected leader, sending a full tablet report...
19:59:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.943192  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 12013
19:59:25.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.957441 11861 connection.cc:573] server connection from 127.9.144.1:33443 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
19:59:25.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.958029  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 11841
19:59:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 19:59:25.959185 12340 negotiation.cc:336] Failed RPC negotiation. Trace:
19:59:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 19:59:25.944677 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.9.144.1:59213 (local address 127.9.144.3:43859)
19:59:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 19:59:25.945279 (+   602us) server_negotiation.cc:207] Beginning negotiation
19:59:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 19:59:25.945289 (+    10us) server_negotiation.cc:400] Waiting for connection header
19:59:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 19:59:25.957833 (+ 12544us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.9.144.1:59213: BlockingRecv error: recv got EOF from 127.9.144.1:59213 (error 108)
19:59:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":430,"spinlock_wait_cycles":1280,"thread_start_us":413,"threads_started":1}
19:59:25.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:25.964547 11967 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.055086  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 11841
19:59:26.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.068675  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 12192
19:59:26.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.076028 12317 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:26.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.164938  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 12192
19:59:26.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.178629  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 9803
19:59:26.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.179757  9864 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:26.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.294536  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 9803
19:59:26.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.311913  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 9871
19:59:26.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.312965  9935 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:26.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.421068  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 9871
19:59:26.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.437901  9792 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu pid 9943
19:59:26.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.439025 10006 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:26.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 19:59:26.549969  9792 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskvUDI3f/build/asan/bin/kudu with pid 9943
19:59:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T19:59:26Z chronyd exiting

Time: 40.055

OK (1 test)

