JUnit version 4.13.2
.10:43:27.488 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1786445007479
10:43:27.553 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
10:43:27.583 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
10:43:27.594 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskA_7F9h/test-tmp
10:43:27.617 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
10:43:27.619 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskA_7F9h/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
10:43:27.814 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

10:43:27.994 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
10:43:27.996 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

10:43:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:27.999929 20627 mini_oidc.cc:150] Starting JWKS server
10:43:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:28.003916 20627 webserver.cc:533] Webserver started at http://127.0.0.1:46079/ using document root <none> and password file <none>
10:43:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:28.004355 20627 mini_oidc.cc:186] Starting OIDC Discovery server
10:43:28.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:28.006479 20627 webserver.cc:533] Webserver started at http://127.0.0.1:37621/ using document root <none> and password file <none>
10:43:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T10:43:28Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
10:43:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T10:43:28Z Disabled control of system clock
10:43:28.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:28.069473 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:28.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:28.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/wal
10:43:28.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/data
10:43:28.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:43:28.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:43:28.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:43:28.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:28.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:28.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:28.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/logs
10:43:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/data/info.pb
10:43:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:43:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:43:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:43:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:43:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:28.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:28.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:28.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:43:28.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:43:28.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:43:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:43:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.254:38609
10:43:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.254
10:43:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:43:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:43:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46079/jwks.json with env {}
10:43:28.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.372167 20645 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:43:28.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.372777 20645 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:43:28.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.373231 20645 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:43:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.406278 20645 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
10:43:28.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.406584 20645 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:43:28.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.406775 20645 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
10:43:28.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.407037 20645 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
10:43:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.407225 20645 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
10:43:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.407521 20645 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46079/jwks.json
10:43:28.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:28.444734 20645 master_runner.cc:387] Master server non-default flags:
10:43:28.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:28.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:28.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:28.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:28.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/data
10:43:28.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/wal
10:43:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:43:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:43:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.254:38609
10:43:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:43:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:43:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:43:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46079/jwks.json
10:43:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/data/info.pb
10:43:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.254
10:43:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:43:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:43:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:43:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/logs
10:43:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:43:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:43:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:43:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:43:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:43:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:43:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:43:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:43:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:43:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:28.446324 20645 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:43:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:28.448045 20645 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:43:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.461136 20660 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:29.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.464560 20658 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:28.467886 20664 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:29.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:29.749217 20662 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1284 milliseconds
10:43:29.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.749348 20645 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:43:29.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.750820 20645 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:43:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.753580 20645 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:43:29.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.755036 20645 hybrid_clock.cc:648] HybridClock initialized: now 1786445009755005 us; error 118 us; skew 500 ppm
10:43:29.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.755980 20645 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:43:29.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.762255 20645 webserver.cc:533] Webserver started at http://127.20.36.254:34653/ using document root <none> and password file <none>
10:43:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.763301 20645 fs_manager.cc:362] Metadata directory not provided
10:43:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.763613 20645 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:43:29.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.764220 20645 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
10:43:29.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.768595 20645 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/data/instance:
10:43:29.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57f3b6777b5c4cfcb653fe008518a302"
10:43:29.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:29 on dist-test-slave-gw5c"
10:43:29.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.769973 20645 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/wal/instance:
10:43:29.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57f3b6777b5c4cfcb653fe008518a302"
10:43:29.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:29 on dist-test-slave-gw5c"
10:43:29.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.777678 20645 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.004s
10:43:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.783365 20673 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:43:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.784662 20645 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
10:43:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.784957 20645 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/wal
10:43:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57f3b6777b5c4cfcb653fe008518a302"
10:43:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:29 on dist-test-slave-gw5c"
10:43:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.785243 20645 fs_report.cc:389] FS layout report
10:43:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:43:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/wal
10:43:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/wal
10:43:29.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/data/data
10:43:29.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:43:29.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:43:29.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:43:29.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:43:29.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:43:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:43:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:43:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:43:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:43:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:43:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:43:29.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.843791 20645 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:43:29.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.845470 20645 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:43:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.845880 20645 kserver.cc:163] Server-wide thread pool size limit: 3276
10:43:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.920183 20645 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.254:38609
10:43:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.920279 20763 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.254:38609 every 8 connection(s)
10:43:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.922930 20645 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/data/info.pb
10:43:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.928033 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 20645
10:43:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.928594 20627 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-0/wal/instance
10:43:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.930262 20765 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:43:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.933784 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/wal
10:43:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/data
10:43:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:43:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:43:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:43:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/logs
10:43:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/data/info.pb
10:43:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:43:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:43:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:43:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:43:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:43:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:43:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:43:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:43:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.253:37673
10:43:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.253
10:43:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:43:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:43:29.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46079/jwks.json with env {}
10:43:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.945439 20765 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } has no permanent_uuid. Determining permanent_uuid...
10:43:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:29.975530 20765 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } has no permanent_uuid. Determining permanent_uuid...
10:43:29.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:29.978308 20677 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.253:37673: connect: Connection refused (error 111)
10:43:29.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:29.983031 20765 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.253:37673: Network error: Client connection negotiation failed: client connection to 127.20.36.253:37673: connect: Connection refused (error 111)
10:43:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:30.032517 20765 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } attempt: 1
10:43:30.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.037595 20765 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.253:37673: Network error: Client connection negotiation failed: client connection to 127.20.36.253:37673: connect: Connection refused (error 111)
10:43:30.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:30.106091 20765 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } attempt: 2
10:43:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.110957 20765 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.253:37673: Network error: Client connection negotiation failed: client connection to 127.20.36.253:37673: connect: Connection refused (error 111)
10:43:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:30.202347 20765 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } attempt: 3
10:43:30.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.206900 20765 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.253:37673: Network error: Client connection negotiation failed: client connection to 127.20.36.253:37673: connect: Connection refused (error 111)
10:43:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.252691 20768 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:43:30.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.253278 20768 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:43:30.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.253886 20768 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:43:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.286499 20768 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
10:43:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.286782 20768 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:43:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.287060 20768 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
10:43:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.287269 20768 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
10:43:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.287531 20768 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
10:43:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.287683 20768 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46079/jwks.json
10:43:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:30.324431 20768 master_runner.cc:387] Master server non-default flags:
10:43:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/data
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/wal
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.253:37673
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46079/jwks.json
10:43:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/data/info.pb
10:43:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.253
10:43:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:43:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:43:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:43:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/logs
10:43:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:43:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:43:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:43:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:43:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:43:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:43:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:43:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:43:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:43:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:30.325968 20768 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:43:30.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:30.327862 20768 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:43:30.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.340977 20777 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:30.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:30.350380 20765 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } attempt: 4
10:43:30.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.363991 20765 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.253:37673: Network error: Client connection negotiation failed: client connection to 127.20.36.253:37673: connect: Connection refused (error 111)
10:43:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:30.663518 20765 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } attempt: 5
10:43:30.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.674167 20765 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.253:37673: Network error: Client connection negotiation failed: client connection to 127.20.36.253:37673: connect: Connection refused (error 111)
10:43:31.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:31.221764 20765 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } attempt: 6
10:43:31.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:31.245891 20765 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.253:37673: Network error: Client connection negotiation failed: client connection to 127.20.36.253:37673: connect: Connection refused (error 111)
10:43:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:31.744323 20776 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 20768
10:43:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:30.341609 20778 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:32.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.152035 20768 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.811s	user 0.631s	sys 1.178s
10:43:32.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.153613 20768 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.813s	user 0.631s	sys 1.179s
10:43:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.155849 20780 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.159162 20779 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1811 milliseconds
10:43:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.159246 20768 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:43:32.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.160712 20768 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:43:32.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.163301 20768 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:43:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.164774 20768 hybrid_clock.cc:648] HybridClock initialized: now 1786445012164739 us; error 39 us; skew 500 ppm
10:43:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.165673 20768 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:43:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.172323 20768 webserver.cc:533] Webserver started at http://127.20.36.253:35987/ using document root <none> and password file <none>
10:43:32.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.173408 20768 fs_manager.cc:362] Metadata directory not provided
10:43:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.173662 20768 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:43:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.174283 20768 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
10:43:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.178805 20768 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/data/instance:
10:43:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71a011cf83004149b6b69f960ab7ece9"
10:43:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:32 on dist-test-slave-gw5c"
10:43:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.180241 20768 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/wal/instance:
10:43:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71a011cf83004149b6b69f960ab7ece9"
10:43:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:32 on dist-test-slave-gw5c"
10:43:32.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.187907 20768 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.001s	sys 0.006s
10:43:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.193814 20799 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:43:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.195003 20768 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.003s
10:43:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.195349 20768 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/wal
10:43:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71a011cf83004149b6b69f960ab7ece9"
10:43:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:32 on dist-test-slave-gw5c"
10:43:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.195794 20768 fs_report.cc:389] FS layout report
10:43:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:43:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/wal
10:43:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/wal
10:43:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/data/data
10:43:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:43:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:43:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:43:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:43:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:43:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:43:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:43:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:43:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:43:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:43:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:43:32.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.263233 20768 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:43:32.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.265501 20768 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:43:32.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.266096 20768 kserver.cc:163] Server-wide thread pool size limit: 3276
10:43:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.306475 20765 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } attempt: 7
10:43:32.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.315861 20765 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.253:37673: Network error: Client connection negotiation failed: client connection to 127.20.36.253:37673: connect: Connection refused (error 111)
10:43:32.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.355137 20768 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.253:37673
10:43:32.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.355228 20879 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.253:37673 every 8 connection(s)
10:43:32.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.358165 20768 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/data/info.pb
10:43:32.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.364154 20880 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:43:32.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.366926 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 20768
10:43:32.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.367339 20627 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-1/wal/instance
10:43:32.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.369450 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/wal
10:43:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/data
10:43:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:43:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:43:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:43:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/logs
10:43:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/data/info.pb
10:43:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:43:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:43:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:43:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:43:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:43:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:43:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:43:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:43:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.252:35339
10:43:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.252
10:43:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:43:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:43:32.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46079/jwks.json with env {}
10:43:32.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.377199 20880 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } has no permanent_uuid. Determining permanent_uuid...
10:43:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.398886 20880 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } has no permanent_uuid. Determining permanent_uuid...
10:43:32.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.414196 20880 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } has no permanent_uuid. Determining permanent_uuid...
10:43:32.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.417104 20802 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.252:35339: connect: Connection refused (error 111)
10:43:32.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.421947 20880 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.252:35339: Network error: Client connection negotiation failed: client connection to 127.20.36.252:35339: connect: Connection refused (error 111)
10:43:32.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.471612 20880 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } attempt: 1
10:43:32.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.476090 20880 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.252:35339: Network error: Client connection negotiation failed: client connection to 127.20.36.252:35339: connect: Connection refused (error 111)
10:43:32.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.544466 20880 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } attempt: 2
10:43:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.548818 20880 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.252:35339: Network error: Client connection negotiation failed: client connection to 127.20.36.252:35339: connect: Connection refused (error 111)
10:43:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.640329 20880 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } attempt: 3
10:43:32.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.645061 20880 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.252:35339: Network error: Client connection negotiation failed: client connection to 127.20.36.252:35339: connect: Connection refused (error 111)
10:43:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.689549 20883 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:43:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.690244 20883 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:43:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.690784 20883 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:43:32.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.724434 20883 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
10:43:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.724722 20883 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:43:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.724915 20883 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
10:43:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.725103 20883 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
10:43:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.725284 20883 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
10:43:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.725520 20883 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46079/jwks.json
10:43:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.761579 20883 master_runner.cc:387] Master server non-default flags:
10:43:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/data
10:43:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/wal
10:43:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:43:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:43:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.252:35339
10:43:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:43:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:43:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:43:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46079/jwks.json
10:43:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/data/info.pb
10:43:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.252
10:43:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:43:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:43:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:43:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/logs
10:43:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.762924 20883 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:43:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.765115 20883 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:43:32.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.778194 20900 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:32.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:32.788489 20880 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } attempt: 4
10:43:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.792949 20880 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.252:35339: Network error: Client connection negotiation failed: client connection to 127.20.36.252:35339: connect: Connection refused (error 111)
10:43:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:33.092423 20880 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } attempt: 5
10:43:33.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:33.099655 20880 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.252:35339: Network error: Client connection negotiation failed: client connection to 127.20.36.252:35339: connect: Connection refused (error 111)
10:43:33.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:33.647306 20880 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } attempt: 6
10:43:33.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:33.667603 20880 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.36.252:35339: Network error: Client connection negotiation failed: client connection to 127.20.36.252:35339: connect: Connection refused (error 111)
10:43:34.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.778656 20901 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:34.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:32.782645 20906 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:34.046934 20905 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1263 milliseconds
10:43:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.047065 20883 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:43:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.048424 20883 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:43:34.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.050946 20883 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:43:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.052347 20883 hybrid_clock.cc:648] HybridClock initialized: now 1786445014052318 us; error 41 us; skew 500 ppm
10:43:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.053241 20883 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:43:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.060197 20883 webserver.cc:533] Webserver started at http://127.20.36.252:34733/ using document root <none> and password file <none>
10:43:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.061251 20883 fs_manager.cc:362] Metadata directory not provided
10:43:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.061502 20883 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:43:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.062058 20883 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
10:43:34.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.066696 20883 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/data/instance:
10:43:34.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b"
10:43:34.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:34 on dist-test-slave-gw5c"
10:43:34.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.068107 20883 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/wal/instance:
10:43:34.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b"
10:43:34.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:34 on dist-test-slave-gw5c"
10:43:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.075853 20883 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.001s
10:43:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.082191 20919 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:43:34.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.083315 20883 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.006s	sys 0.000s
10:43:34.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.083766 20883 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/wal
10:43:34.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b"
10:43:34.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:34 on dist-test-slave-gw5c"
10:43:34.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.084198 20883 fs_report.cc:389] FS layout report
10:43:34.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:43:34.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/wal
10:43:34.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/wal
10:43:34.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/data/data
10:43:34.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:43:34.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:43:34.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:43:34.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:43:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:43:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:43:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:43:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:43:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:43:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:43:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:43:34.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.159282 20883 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:43:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.161085 20883 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:43:34.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.161685 20883 kserver.cc:163] Server-wide thread pool size limit: 3276
10:43:34.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.240628 20883 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.252:35339
10:43:34.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.240708 21005 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.252:35339 every 8 connection(s)
10:43:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.243746 20883 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/data/info.pb
10:43:34.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.249857 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 20883
10:43:34.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.249998 21007 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:43:34.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.250315 20627 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/master-2/wal/instance
10:43:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.264645 21007 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } has no permanent_uuid. Determining permanent_uuid...
10:43:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.291821 21007 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } has no permanent_uuid. Determining permanent_uuid...
10:43:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.302266 21007 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } has no permanent_uuid. Determining permanent_uuid...
10:43:34.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.318500 21007 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b: Bootstrap starting.
10:43:34.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.325532 21007 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b: Neither blocks nor log segments found. Creating new log.
10:43:34.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.327472 21007 log.cc:826] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b: Log is configured to *not* fsync() on all Append() calls
10:43:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.333227 21007 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b: No bootstrap required, opened a new log
10:43:34.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.352754 21007 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } }
10:43:34.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.353526 21007 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:43:34.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.353809 21007 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9cd0f87a3a8e44fca4eefa5e2a71228b, State: Initialized, Role: FOLLOWER
10:43:34.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.355157 21007 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } }
10:43:34.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.358435 21019 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } } }
10:43:34.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.359302 21019 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [sys.catalog]: This master's current role is: FOLLOWER
10:43:34.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.361627 21007 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [sys.catalog]: configured and running, proceeding with master startup.
10:43:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:34.391731 21036 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:43:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:34.392016 21036 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:43:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.406355 20765 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } attempt: 8
10:43:34.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.388720 20883 master_runner.cc:187] Error getting master registration for 127.20.36.254:38609: OK, instance_id { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" instance_seqno: 1786445009900266 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:43:34.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.415990 20883 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
10:43:34.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:34.417801 20931 tablet.cc:2406] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:43:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.422959 20765 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } has no permanent_uuid. Determining permanent_uuid...
10:43:34.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.439276 20765 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302: Bootstrap starting.
10:43:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.444795 20765 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302: Neither blocks nor log segments found. Creating new log.
10:43:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.446486 20765 log.cc:826] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302: Log is configured to *not* fsync() on all Append() calls
10:43:34.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.451841 20765 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302: No bootstrap required, opened a new log
10:43:34.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.472471 20765 raft_consensus.cc:359] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } }
10:43:34.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.473146 20765 raft_consensus.cc:385] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:43:34.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.473443 20765 raft_consensus.cc:740] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57f3b6777b5c4cfcb653fe008518a302, State: Initialized, Role: FOLLOWER
10:43:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.474388 20765 consensus_queue.cc:260] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } }
10:43:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.477210 21043 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } } }
10:43:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.478048 21043 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [sys.catalog]: This master's current role is: FOLLOWER
10:43:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.478574 20765 sys_catalog.cc:565] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [sys.catalog]: configured and running, proceeding with master startup.
10:43:34.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:34.513845 21062 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:43:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:34.514699 21062 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:43:34.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.513602 20645 master_runner.cc:187] Error getting master registration for 127.20.36.253:37673: OK, instance_id { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" instance_seqno: 1786445012334394 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:43:34.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.549592 20645 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
10:43:34.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:34.608855 20681 tablet.cc:2406] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:43:34.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.728186 20880 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } attempt: 7
10:43:34.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.749660 20880 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9: Bootstrap starting.
10:43:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.755692 20880 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9: Neither blocks nor log segments found. Creating new log.
10:43:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.757681 20880 log.cc:826] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9: Log is configured to *not* fsync() on all Append() calls
10:43:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.762657 20880 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9: No bootstrap required, opened a new log
10:43:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.762796 21043 raft_consensus.cc:493] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:43:34.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.763309 21043 raft_consensus.cc:515] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } }
10:43:34.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.766093 21043 leader_election.cc:290] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 71a011cf83004149b6b69f960ab7ece9 (127.20.36.253:37673), 9cd0f87a3a8e44fca4eefa5e2a71228b (127.20.36.252:35339)
10:43:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.769071 20970 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "57f3b6777b5c4cfcb653fe008518a302" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" is_pre_election: true
10:43:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.770002 20970 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57f3b6777b5c4cfcb653fe008518a302 in term 0.
10:43:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.776871 20677 leader_election.cc:304] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 57f3b6777b5c4cfcb653fe008518a302, 9cd0f87a3a8e44fca4eefa5e2a71228b; no voters: 
10:43:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.781657 21043 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:43:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.793277 21043 raft_consensus.cc:493] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:43:34.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.794127 21043 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 0 FOLLOWER]: Advancing to term 1
10:43:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.802444 21043 raft_consensus.cc:515] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } }
10:43:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.809007 20970 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "57f3b6777b5c4cfcb653fe008518a302" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b"
10:43:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.809614 20970 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [term 0 FOLLOWER]: Advancing to term 1
10:43:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.814049 20880 raft_consensus.cc:359] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } }
10:43:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.815487 20880 raft_consensus.cc:385] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:43:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.816011 20880 raft_consensus.cc:740] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 71a011cf83004149b6b69f960ab7ece9, State: Initialized, Role: FOLLOWER
10:43:34.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.818502 20880 consensus_queue.cc:260] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } }
10:43:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.834036 21043 leader_election.cc:290] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [CANDIDATE]: Term 1 election: Requested vote from peers 71a011cf83004149b6b69f960ab7ece9 (127.20.36.253:37673), 9cd0f87a3a8e44fca4eefa5e2a71228b (127.20.36.252:35339)
10:43:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.837306 20970 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57f3b6777b5c4cfcb653fe008518a302 in term 1.
10:43:34.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.855253 20677 leader_election.cc:304] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 57f3b6777b5c4cfcb653fe008518a302, 9cd0f87a3a8e44fca4eefa5e2a71228b; no voters: 
10:43:34.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.856792 21043 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 1 FOLLOWER]: Leader election won for term 1
10:43:34.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.871953 20880 sys_catalog.cc:565] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [sys.catalog]: configured and running, proceeding with master startup.
10:43:34.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.872135 20846 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "57f3b6777b5c4cfcb653fe008518a302" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "71a011cf83004149b6b69f960ab7ece9" is_pre_election: true
10:43:34.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.873513 21043 raft_consensus.cc:697] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [term 1 LEADER]: Becoming Leader. State: Replica: 57f3b6777b5c4cfcb653fe008518a302, State: Running, Role: LEADER
10:43:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.874527 21043 consensus_queue.cc:237] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } }
10:43:34.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.872829 20845 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "57f3b6777b5c4cfcb653fe008518a302" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "71a011cf83004149b6b69f960ab7ece9"
10:43:34.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.876363 20846 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57f3b6777b5c4cfcb653fe008518a302 in term 0.
10:43:34.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.881243 21066 sys_catalog.cc:455] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } } }
10:43:34.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.885329 21066 sys_catalog.cc:458] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [sys.catalog]: This master's current role is: FOLLOWER
10:43:34.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.893643 21068 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 57f3b6777b5c4cfcb653fe008518a302. Latest consensus state: current_term: 1 leader_uuid: "57f3b6777b5c4cfcb653fe008518a302" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } } }
10:43:34.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.896225 21068 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [sys.catalog]: This master's current role is: LEADER
10:43:34.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.912776 21072 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
10:43:34.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.920850 21072 catalog_manager.cc:1520] Initializing Kudu cluster ID...
10:43:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.942782 20970 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [term 1 FOLLOWER]: Refusing update from remote peer 57f3b6777b5c4cfcb653fe008518a302: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:43:34.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.944531 20845 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [term 0 FOLLOWER]: Advancing to term 1
10:43:34.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.946420 21068 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:43:34.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.956005 20845 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [term 1 FOLLOWER]: Refusing update from remote peer 57f3b6777b5c4cfcb653fe008518a302: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:43:34.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:34.959040 21068 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [LEADER]: Connected to new peer: Peer: permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:43:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.001158 21066 sys_catalog.cc:455] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 57f3b6777b5c4cfcb653fe008518a302. Latest consensus state: current_term: 1 leader_uuid: "57f3b6777b5c4cfcb653fe008518a302" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } } }
10:43:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.002384 21066 sys_catalog.cc:458] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [sys.catalog]: This master's current role is: FOLLOWER
10:43:35.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.020900 21081 mvcc.cc:204] Tried to move back new op lower bound from 7317278781184540672 to 7317278780959604736. Current Snapshot: MvccSnapshot[applied={T|T < 7317278781184540672}]
10:43:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.023924 21088 mvcc.cc:204] Tried to move back new op lower bound from 7317278781184540672 to 7317278780959604736. Current Snapshot: MvccSnapshot[applied={T|T < 7317278781184540672}]
10:43:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.044819 21066 sys_catalog.cc:455] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "57f3b6777b5c4cfcb653fe008518a302" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } } }
10:43:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.045553 21066 sys_catalog.cc:458] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9 [sys.catalog]: This master's current role is: FOLLOWER
10:43:35.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.051661 21043 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "57f3b6777b5c4cfcb653fe008518a302" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } } }
10:43:35.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.052371 21043 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [sys.catalog]: This master's current role is: LEADER
10:43:35.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.056524 21084 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 57f3b6777b5c4cfcb653fe008518a302. Latest consensus state: current_term: 1 leader_uuid: "57f3b6777b5c4cfcb653fe008518a302" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } } }
10:43:35.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.057320 21084 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [sys.catalog]: This master's current role is: FOLLOWER
10:43:35.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.057610 21072 catalog_manager.cc:1383] Generated new cluster ID: ecd9196c48044df2aeaebdb728930a9f
10:43:35.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.057890 21072 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
10:43:35.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.067574 21068 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "57f3b6777b5c4cfcb653fe008518a302" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } } }
10:43:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.069543 21068 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302 [sys.catalog]: This master's current role is: LEADER
10:43:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.071164 21084 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "57f3b6777b5c4cfcb653fe008518a302" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57f3b6777b5c4cfcb653fe008518a302" member_type: VOTER last_known_addr { host: "127.20.36.254" port: 38609 } } peers { permanent_uuid: "71a011cf83004149b6b69f960ab7ece9" member_type: VOTER last_known_addr { host: "127.20.36.253" port: 37673 } } peers { permanent_uuid: "9cd0f87a3a8e44fca4eefa5e2a71228b" member_type: VOTER last_known_addr { host: "127.20.36.252" port: 35339 } } }
10:43:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.071888 21084 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b [sys.catalog]: This master's current role is: FOLLOWER
10:43:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.084597 21101 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
10:43:35.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.141821 21072 catalog_manager.cc:1406] Generated new certificate authority record
10:43:35.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.144012 21072 catalog_manager.cc:1540] Loading token signing keys...
10:43:35.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.177735 21072 catalog_manager.cc:6092] T 00000000000000000000000000000000 P 57f3b6777b5c4cfcb653fe008518a302: Generated new TSK 0
10:43:35.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.179502 21072 catalog_manager.cc:1550] Initializing in-progress tserver states...
10:43:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.223804 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:43:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data
10:43:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:43:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/logs
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.193:0
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.193
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.193
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:43:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.397972 21036 catalog_manager.cc:1295] Loaded cluster ID: ecd9196c48044df2aeaebdb728930a9f
10:43:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.398901 21036 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b: loading cluster ID for follower catalog manager: success
10:43:35.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.406909 21036 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b: acquiring CA information for follower catalog manager: success
10:43:35.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.423913 21036 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 9cd0f87a3a8e44fca4eefa5e2a71228b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:43:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:35.705838 21107 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:43:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:35.706513 21107 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:43:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:35.707217 21107 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:43:35.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:35.754523 21107 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:43:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:35.755878 21107 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.193
10:43:35.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.794914 21107 tablet_server_runner.cc:78] Tablet server non-default flags:
10:43:35.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:35.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.193:0
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:43:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.193
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.193
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/logs
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:43:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:43:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:43:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:43:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:43:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:43:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:43:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:43:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:43:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.796813 21107 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:43:35.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:35.799144 21107 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:43:35.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:35.815771 21124 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:36.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:36.115869 21101 catalog_manager.cc:1295] Loaded cluster ID: ecd9196c48044df2aeaebdb728930a9f
10:43:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:36.116633 21101 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9: loading cluster ID for follower catalog manager: success
10:43:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:36.138139 21101 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9: acquiring CA information for follower catalog manager: success
10:43:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:36.165491 21101 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 71a011cf83004149b6b69f960ab7ece9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:43:38.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:35.816151 21122 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:38.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:37.216419 21119 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 21107
10:43:38.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:38.226182 21125 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2407 milliseconds
10:43:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:38.226645 21127 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:38.224329 21107 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.407s	user 0.653s	sys 1.182s
10:43:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:38.227176 21107 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.410s	user 0.654s	sys 1.182s
10:43:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.227507 21107 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:43:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.233546 21107 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:43:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.236105 21107 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:43:38.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.237493 21107 hybrid_clock.cc:648] HybridClock initialized: now 1786445018237450 us; error 65 us; skew 500 ppm
10:43:38.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.238384 21107 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:43:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.246099 21107 webserver.cc:533] Webserver started at http://127.20.36.193:35283/ using document root <none> and password file <none>
10:43:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.247123 21107 fs_manager.cc:362] Metadata directory not provided
10:43:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.247378 21107 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:43:38.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.247967 21107 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
10:43:38.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.252699 21107 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/instance:
10:43:38.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdd21cb83af541e895b0058bf4e0b6d2"
10:43:38.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:38 on dist-test-slave-gw5c"
10:43:38.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.253923 21107 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal/instance:
10:43:38.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdd21cb83af541e895b0058bf4e0b6d2"
10:43:38.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:38 on dist-test-slave-gw5c"
10:43:38.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.261972 21107 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.001s
10:43:38.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.268870 21139 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:43:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.270226 21107 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.003s
10:43:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.270591 21107 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:43:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdd21cb83af541e895b0058bf4e0b6d2"
10:43:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:38 on dist-test-slave-gw5c"
10:43:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.270968 21107 fs_report.cc:389] FS layout report
10:43:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:43:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:43:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/data
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:43:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:43:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.324589 21107 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:43:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.326259 21107 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:43:38.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.326794 21107 kserver.cc:163] Server-wide thread pool size limit: 3276
10:43:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.329782 21107 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:43:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.334921 21107 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:43:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.335160 21107 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
10:43:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.335547 21107 ts_tablet_manager.cc:616] Registered 0 tablets
10:43:38.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.335772 21107 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
10:43:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.552044 21107 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.193:34951
10:43:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.552170 21330 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.193:34951 every 8 connection(s)
10:43:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.555799 21107 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:43:38.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.566560 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 21107
10:43:38.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.567209 20627 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal/instance
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.581535 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/logs
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.194:0
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.194
10:43:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.194
10:43:38.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:38.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:38.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:38.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:38.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:38.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:43:38.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.612108 21336 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:43:38.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.612749 21336 heartbeater.cc:461] Registering TS with master...
10:43:38.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.614408 21336 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:43:38.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.615975 21331 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:43:38.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.616380 21331 heartbeater.cc:461] Registering TS with master...
10:43:38.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.617422 21331 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:43:38.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.618975 21339 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:43:38.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.619315 21339 heartbeater.cc:461] Registering TS with master...
10:43:38.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.620255 21339 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:43:38.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.623319 20951 ts_manager.cc:194] Registered new tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:38.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.623481 20699 ts_manager.cc:194] Registered new tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:38.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.623869 20825 ts_manager.cc:194] Registered new tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:38.627025 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.193:41391
10:43:38.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:38.939534 21354 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:43:38.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:38.940138 21354 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:43:38.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:38.940822 21354 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:43:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:38.976461 21354 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:43:38.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:38.977502 21354 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.194
10:43:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:39.015173 21354 tablet_server_runner.cc:78] Tablet server non-default flags:
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.194:0
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:43:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.194
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.194
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/logs
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:43:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:43:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:43:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:43:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:43:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:43:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:43:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:43:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:43:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:39.017113 21354 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:43:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:39.019623 21354 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:43:39.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:39.035921 21360 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:39.631423 21339 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:43:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:39.037276 21361 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:40.438899 21359 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 21354
10:43:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:40.648931 21364 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1609 milliseconds
10:43:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:40.647785 21354 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.610s	user 0.002s	sys 0.000s
10:43:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:40.649780 21354 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.612s	user 0.002s	sys 0.000s
10:43:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.650398 21354 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:43:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:40.650480 21366 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:40.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.654067 21354 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:43:40.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.656752 21354 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:43:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.658169 21354 hybrid_clock.cc:648] HybridClock initialized: now 1786445020658135 us; error 39 us; skew 500 ppm
10:43:40.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.658926 21354 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:43:40.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.665088 21354 webserver.cc:533] Webserver started at http://127.20.36.194:45775/ using document root <none> and password file <none>
10:43:40.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.665944 21354 fs_manager.cc:362] Metadata directory not provided
10:43:40.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.666129 21354 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:43:40.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.666515 21354 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
10:43:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.670703 21354 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/instance:
10:43:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:43:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:40 on dist-test-slave-gw5c"
10:43:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.671813 21354 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal/instance:
10:43:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:43:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:40 on dist-test-slave-gw5c"
10:43:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.679162 21354 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.000s
10:43:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.684914 21383 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:43:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.686002 21354 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.003s
10:43:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.686363 21354 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:43:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:43:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:40 on dist-test-slave-gw5c"
10:43:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.686750 21354 fs_report.cc:389] FS layout report
10:43:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:43:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:43:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:43:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/data
10:43:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:43:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:43:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:43:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:43:40.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:43:40.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:43:40.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:43:40.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:43:40.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:43:40.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:43:40.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:43:40.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.735409 21354 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:43:40.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.737016 21354 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:43:40.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.737408 21354 kserver.cc:163] Server-wide thread pool size limit: 3276
10:43:40.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.740080 21354 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:43:40.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.744166 21354 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:43:40.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.744357 21354 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
10:43:40.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.744578 21354 ts_tablet_manager.cc:616] Registered 0 tablets
10:43:40.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.744709 21354 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
10:43:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.891144 21354 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.194:39729
10:43:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.891249 21568 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.194:39729 every 8 connection(s)
10:43:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.894028 21354 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:43:40.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.902658 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 21354
10:43:40.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.903261 20627 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal/instance
10:43:40.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.934134 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/logs
10:43:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.195:0
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.195
10:43:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.195
10:43:40.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:40.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:40.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:40.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:40.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:40.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:43:40.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.960641 21569 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:43:40.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.960877 21571 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:43:40.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.961184 21569 heartbeater.cc:461] Registering TS with master...
10:43:40.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.961308 21571 heartbeater.cc:461] Registering TS with master...
10:43:40.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.963045 21571 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:43:40.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.963565 21569 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:43:40.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.966463 20951 ts_manager.cc:194] Registered new tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:43:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.967844 20699 ts_manager.cc:194] Registered new tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:43:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.969564 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.194:41539
10:43:40.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.971005 21570 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:43:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.971311 21570 heartbeater.cc:461] Registering TS with master...
10:43:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.972569 21570 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:43:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:40.975142 20825 ts_manager.cc:194] Registered new tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:43:41.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:41.301268 21586 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:43:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:41.301774 21586 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:43:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:41.302372 21586 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:43:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:41.334609 21586 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:43:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:41.335642 21586 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.195
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:41.388845 21586 tablet_server_runner.cc:78] Tablet server non-default flags:
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.195:0
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:43:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.195
10:43:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:43:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.195
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/logs
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:43:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:43:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:41.390571 21586 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:43:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:41.392666 21586 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:43:41.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:41.406690 21595 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:41.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:41.973402 21571 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:43:42.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:41.409011 21597 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:42.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:42.629143 21598 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
10:43:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:42.630388 21599 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.630391 21586 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:43:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.634078 21586 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:43:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.636471 21586 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:43:42.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.637871 21586 hybrid_clock.cc:648] HybridClock initialized: now 1786445022637840 us; error 33 us; skew 500 ppm
10:43:42.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.638731 21586 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:43:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.645280 21586 webserver.cc:533] Webserver started at http://127.20.36.195:44795/ using document root <none> and password file <none>
10:43:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.646270 21586 fs_manager.cc:362] Metadata directory not provided
10:43:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.646526 21586 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:43:42.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.647078 21586 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
10:43:42.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.654633 21586 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/instance:
10:43:42.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a766ec61926c4e27b86cde01c49ea7ed"
10:43:42.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:42 on dist-test-slave-gw5c"
10:43:42.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.656096 21586 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal/instance:
10:43:42.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a766ec61926c4e27b86cde01c49ea7ed"
10:43:42.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:42 on dist-test-slave-gw5c"
10:43:42.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.663981 21586 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.001s
10:43:42.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.672021 21615 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:43:42.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.673457 21586 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.002s
10:43:42.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.673883 21586 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:43:42.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a766ec61926c4e27b86cde01c49ea7ed"
10:43:42.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:42 on dist-test-slave-gw5c"
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.674351 21586 fs_report.cc:389] FS layout report
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/data
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:43:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:43:42.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.735883 21586 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:43:42.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.737417 21586 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:43:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.737834 21586 kserver.cc:163] Server-wide thread pool size limit: 3276
10:43:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.740605 21586 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:43:42.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.745024 21586 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:43:42.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.745270 21586 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
10:43:42.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.745585 21586 ts_tablet_manager.cc:616] Registered 0 tablets
10:43:42.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.745796 21586 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
10:43:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.892839 21586 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.195:44799
10:43:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.893013 21809 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.195:44799 every 8 connection(s)
10:43:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.895669 21586 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:43:42.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.898911 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 21586
10:43:42.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.899417 20627 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal/instance
10:43:42.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.941998 21810 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:43:42.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.942473 21810 heartbeater.cc:461] Registering TS with master...
10:43:42.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.943948 21810 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:43:42.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.948578 20951 ts_manager.cc:194] Registered new tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:43:42.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.952703 21812 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:43:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.953053 21812 heartbeater.cc:461] Registering TS with master...
10:43:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.954002 21812 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:43:42.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.956621 20825 ts_manager.cc:194] Registered new tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:43:42.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.957964 21814 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:43:42.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.958225 21814 heartbeater.cc:461] Registering TS with master...
10:43:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.959054 21814 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:43:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.961145 20699 ts_manager.cc:194] Registered new tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:43:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.962450 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.195:41615
10:43:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:42.972301 20627 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
10:43:42.985 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
10:43:42.986 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

10:43:42.995 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.20.36.254"
      port: 38609
    }
    bound_http_address {
      host: "127.20.36.254"
      port: 34653
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.20.36.253"
      port: 37673
    }
    bound_http_address {
      host: "127.20.36.253"
      port: 35987
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.20.36.252"
      port: 35339
    }
    bound_http_address {
      host: "127.20.36.252"
      port: 34733
    }
  }
}

10:43:42.999 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

10:43:43.001 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.20.36.193"
      port: 34951
    }
    bound_http_address {
      host: "127.20.36.193"
      port: 35283
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.20.36.194"
      port: 39729
    }
    bound_http_address {
      host: "127.20.36.194"
      port: 45775
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.20.36.195"
      port: 44799
    }
    bound_http_address {
      host: "127.20.36.195"
      port: 44795
    }
  }
}

10:43:43.001 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
10:43:43.198 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
10:43:43.265 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
10:43:43.437 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.36.252:35339(127.20.36.252:35339)] connecting to peer
10:43:43.496 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.36.253:37673(127.20.36.253:37673)] connecting to peer
10:43:43.498 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.36.254:38609(127.20.36.254:38609)] connecting to peer
10:43:43.517 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.36.253:37673(127.20.36.253:37673)] Successfully connected to peer
10:43:43.527 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.20.36.254:38609(127.20.36.254:38609)] Successfully connected to peer
10:43:43.531 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.36.252:35339(127.20.36.252:35339)] Successfully connected to peer
10:43:43.632 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.20.36.252
10:43:43.633 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.20.36.254
10:43:43.633 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.20.36.253
10:43:43.858 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x434348ad, L:/127.0.0.1:33890 - R:/127.20.36.254:38609] using SASL/PLAIN
10:43:43.858 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x4de4dde7, L:/127.0.0.1:40004 - R:/127.20.36.253:37673] using SASL/PLAIN
10:43:43.863 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x2f9379cb, L:/127.0.0.1:40306 - R:/127.20.36.252:35339] using SASL/PLAIN
10:43:43.895 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:43:43.896 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=501), Traces: [0ms] refreshing cache from master, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.36.252:35339, [93ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.36.253:37673, [94ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.36.254:38609, [471ms] Sub RPC ConnectToMaster: received response from server master-127.20.36.253:37673: OK, [471ms] Sub RPC ConnectToMaster: received response from server master-127.20.36.252:35339: OK, [471ms] Sub RPC ConnectToMaster: received response from server master-127.20.36.254:38609: OK, deferred=null) after lookup
10:43:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:43.949797 20699 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33890:
10:43:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379"
10:43:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
10:43:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
10:43:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
10:43:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
10:43:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
10:43:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
10:43:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
10:43:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
10:43:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
10:43:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
10:43:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
10:43:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
10:43:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:43.954478 20699 catalog_manager.cc:7099] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
10:43:43.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:43.966238 21814 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:43:44.059 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 2cb0f8eeb5ec424c9a59700024bce79c
10:43:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.079217 21699 tablet_service.cc:1511] Processing CreateTablet for tablet d339d350474b44a98222592091597860 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379 [id=2cb0f8eeb5ec424c9a59700024bce79c]), partition=RANGE (key) PARTITION UNBOUNDED
10:43:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.082033 21699 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d339d350474b44a98222592091597860. 1 dirs total, 0 dirs full, 0 dirs failed
10:43:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.085120 21229 tablet_service.cc:1511] Processing CreateTablet for tablet d339d350474b44a98222592091597860 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379 [id=2cb0f8eeb5ec424c9a59700024bce79c]), partition=RANGE (key) PARTITION UNBOUNDED
10:43:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.087205 21229 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d339d350474b44a98222592091597860. 1 dirs total, 0 dirs full, 0 dirs failed
10:43:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.087184 21465 tablet_service.cc:1511] Processing CreateTablet for tablet d339d350474b44a98222592091597860 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379 [id=2cb0f8eeb5ec424c9a59700024bce79c]), partition=RANGE (key) PARTITION UNBOUNDED
10:43:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.099611 21465 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d339d350474b44a98222592091597860. 1 dirs total, 0 dirs full, 0 dirs failed
10:43:44.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.120004 21874 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap starting.
10:43:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.130324 21875 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Bootstrap starting.
10:43:44.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.132054 21874 tablet_bootstrap.cc:654] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Neither blocks nor log segments found. Creating new log.
10:43:44.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.134758 21874 log.cc:826] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Log is configured to *not* fsync() on all Append() calls
10:43:44.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.139070 21875 tablet_bootstrap.cc:654] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Neither blocks nor log segments found. Creating new log.
10:43:44.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.139288 21876 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Bootstrap starting.
10:43:44.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.142617 21875 log.cc:826] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Log is configured to *not* fsync() on all Append() calls
10:43:44.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.147203 21874 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: No bootstrap required, opened a new log
10:43:44.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.148278 21874 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Time spent bootstrapping tablet: real 0.029s	user 0.023s	sys 0.000s
10:43:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.151635 21876 tablet_bootstrap.cc:654] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Neither blocks nor log segments found. Creating new log.
10:43:44.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.154215 21876 log.cc:826] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Log is configured to *not* fsync() on all Append() calls
10:43:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.167217 21875 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: No bootstrap required, opened a new log
10:43:44.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.167820 21875 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Time spent bootstrapping tablet: real 0.038s	user 0.023s	sys 0.010s
10:43:44.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.170974 21876 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: No bootstrap required, opened a new log
10:43:44.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.172266 21876 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Time spent bootstrapping tablet: real 0.034s	user 0.006s	sys 0.014s
10:43:44.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.181691 21874 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:44.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.182979 21874 raft_consensus.cc:385] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:43:44.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.183504 21874 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a766ec61926c4e27b86cde01c49ea7ed, State: Initialized, Role: FOLLOWER
10:43:44.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.184755 21874 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:44.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.197571 21874 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Time spent starting tablet: real 0.049s	user 0.034s	sys 0.008s
10:43:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.198732 21875 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.199975 21875 raft_consensus.cc:385] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:43:44.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.200284 21875 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdd21cb83af541e895b0058bf4e0b6d2, State: Initialized, Role: FOLLOWER
10:43:44.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.201392 21875 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:44.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.205569 21875 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Time spent starting tablet: real 0.037s	user 0.029s	sys 0.003s
10:43:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.231077 21876 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.231871 21876 raft_consensus.cc:385] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:43:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.232158 21876 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5373a0a37e14c38bc09e40a3616b123, State: Initialized, Role: FOLLOWER
10:43:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.233520 21876 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:44.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.237735 21876 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Time spent starting tablet: real 0.065s	user 0.026s	sys 0.008s
10:43:44.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:44.337404 21342 tablet.cc:2406] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:43:44.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:44.436772 21819 tablet.cc:2406] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:43:44.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.443547 21880 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:43:44.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.444082 21880 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:44.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.446641 21880 leader_election.cc:290] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:44.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.459466 21493 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "a766ec61926c4e27b86cde01c49ea7ed" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c5373a0a37e14c38bc09e40a3616b123" is_pre_election: true
10:43:44.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.460505 21493 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a766ec61926c4e27b86cde01c49ea7ed in term 0.
10:43:44.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:44.462159 21572 tablet.cc:2406] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:43:44.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.462821 21616 leader_election.cc:304] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, c5373a0a37e14c38bc09e40a3616b123; no voters: 
10:43:44.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.463798 21880 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 0 FOLLOWER]: Leader pre-election won for term 1
10:43:44.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.464192 21880 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:43:44.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.464514 21880 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 0 FOLLOWER]: Advancing to term 1
10:43:44.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.468194 21259 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "a766ec61926c4e27b86cde01c49ea7ed" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" is_pre_election: true
10:43:44.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.469120 21259 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a766ec61926c4e27b86cde01c49ea7ed in term 0.
10:43:44.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.469616 21880 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:44.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.471740 21880 leader_election.cc:290] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 1 election: Requested vote from peers c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:44.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.472509 21493 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "a766ec61926c4e27b86cde01c49ea7ed" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:43:44.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.472754 21259 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "a766ec61926c4e27b86cde01c49ea7ed" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cdd21cb83af541e895b0058bf4e0b6d2"
10:43:44.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.473019 21493 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 0 FOLLOWER]: Advancing to term 1
10:43:44.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.473289 21259 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 0 FOLLOWER]: Advancing to term 1
10:43:44.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.478005 21259 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a766ec61926c4e27b86cde01c49ea7ed in term 1.
10:43:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.479377 21616 leader_election.cc:304] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, cdd21cb83af541e895b0058bf4e0b6d2; no voters: 
10:43:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.480119 21880 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 1 FOLLOWER]: Leader election won for term 1
10:43:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.480720 21493 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a766ec61926c4e27b86cde01c49ea7ed in term 1.
10:43:44.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.482511 21880 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 1 LEADER]: Becoming Leader. State: Replica: a766ec61926c4e27b86cde01c49ea7ed, State: Running, Role: LEADER
10:43:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.483541 21880 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:44.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.498778 20698 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed reported cstate change: term changed from 0 to 1, leader changed from <none> to a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195). New cstate: current_term: 1 leader_uuid: "a766ec61926c4e27b86cde01c49ea7ed" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: UNKNOWN } } }
10:43:44.544 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 2cb0f8eeb5ec424c9a59700024bce79c
10:43:44.564 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:44.571 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299993, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:44.573 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [9ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=null) after lookup
10:43:44.586 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:43:44.589 [DEBUG - kudu-nio-3] (Connection.java:757) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] Successfully connected to peer
10:43:44.661 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0xb9a71c16, L:/127.0.0.1:54870 - R:/127.20.36.195:44799] using TOKEN/null
10:43:44.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.675364 21493 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 1 FOLLOWER]: Refusing update from remote peer a766ec61926c4e27b86cde01c49ea7ed: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:43:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.675361 21259 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 1 FOLLOWER]: Refusing update from remote peer a766ec61926c4e27b86cde01c49ea7ed: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:43:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.677079 21880 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [LEADER]: Connected to new peer: Peer: permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:43:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.677709 21888 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [LEADER]: Connected to new peer: Peer: permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:43:44.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.706086 21893 mvcc.cc:204] Tried to move back new op lower bound from 7317278821053583360 to 7317278820293251072. Current Snapshot: MvccSnapshot[applied={T|T < 7317278821053583360}]
10:43:44.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.707721 21894 mvcc.cc:204] Tried to move back new op lower bound from 7317278821053583360 to 7317278820293251072. Current Snapshot: MvccSnapshot[applied={T|T < 7317278821053583360}]
10:43:44.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.714354 21897 mvcc.cc:204] Tried to move back new op lower bound from 7317278821053583360 to 7317278820293251072. Current Snapshot: MvccSnapshot[applied={T|T < 7317278821053583360}]
10:43:44.837 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:43:44.838 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.36.195:44799
10:43:44.840 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

10:43:44.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:44.841032 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 21586
10:43:44.883 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:43:44.885 [DEBUG - kudu-nio-3] (Connection.java:239) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] handling channelInactive
10:43:44.886 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state READY due to: connection closed
10:43:44.886 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: connection closed
10:43:44.905 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:44.905 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:44.905 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [3ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@570558580(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@eba7d4d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a350d9c -> wakeup thread Time-limited test)) after lookup
10:43:44.906 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:43:44.909 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:44.909 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:44.910 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:44.925 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:44.925 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:44.926 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [3ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [27ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [27ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@570558580(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@eba7d4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a350d9c -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843))) after lookup
10:43:44.927 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:43:44.929 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:44.929 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:44.929 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:44.965 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:44.965 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:44.966 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=82), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [3ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [27ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [27ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [46ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [46ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@570558580(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@eba7d4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843) -> (continuation of Deferred@680613589 after retry RPC@413456673), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a350d9c -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843) -> (continuation of Deferred@680613589 after retry RPC@413456673))) after lookup
10:43:44.966 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:43:44.968 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:44.968 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:44.969 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:45.045 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:45.045 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:45.046 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=162), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [3ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [27ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [27ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [46ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [46ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [83ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [86ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [86ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [161ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@570558580(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@eba7d4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843) -> (continuation of Deferred@680613589 after retry RPC@413456673) -> (continuation of Deferred@1023605853 after retry RPC@370802624), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a350d9c -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843) -> (continuation of Deferred@680613589 after retry RPC@413456673) -> (continuation of Deferred@1023605853 after retry RPC@370802624))) after lookup
10:43:45.046 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:43:45.048 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:45.048 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:45.048 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:45.505 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:45.505 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:45.506 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=622), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [3ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [27ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [27ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [46ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [46ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [83ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [86ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [86ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [161ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [163ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [165ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [165ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [617ms] refreshing cache from master, [617ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [622ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@570558580(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@eba7d4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843) -> (continuation of Deferred@680613589 after retry RPC@413456673) -> (continuation of Deferred@1023605853 after retry RPC@370802624) -> (continuation of Deferred@1238989935 after retry RPC@365226792), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a350d9c -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843) -> (continuation of Deferred@680613589 after retry RPC@413456673) -> (continuation of Deferred@1023605853 after retry RPC@370802624) -> (continuation of Deferred@1238989935 after retry RPC@365226792))) after lookup
10:43:45.507 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:43:45.508 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:45.508 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:45.508 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:45.905 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:45.906 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:45.907 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1023), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [3ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [27ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [27ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [46ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [46ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [83ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [86ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [86ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [161ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [163ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [165ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [165ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [617ms] refreshing cache from master, [617ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [622ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [623ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [625ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [626ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1018ms] refreshing cache from master, [1018ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [1022ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@570558580(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@eba7d4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843) -> (continuation of Deferred@680613589 after retry RPC@413456673) -> (continuation of Deferred@1023605853 after retry RPC@370802624) -> (continuation of Deferred@1238989935 after retry RPC@365226792) -> (continuation of Deferred@1041915802 after retry RPC@213216986), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a350d9c -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843) -> (continuation of Deferred@680613589 after retry RPC@413456673) -> (continuation of Deferred@1023605853 after retry RPC@370802624) -> (continuation of Deferred@1238989935 after retry RPC@365226792) -> (continuation of Deferred@1041915802 after retry RPC@213216986))) after lookup
10:43:45.908 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:43:45.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:45.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:45.909 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:43:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.288151 21919 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader a766ec61926c4e27b86cde01c49ea7ed)
10:43:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.288803 21920 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader a766ec61926c4e27b86cde01c49ea7ed)
10:43:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.289167 21920 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.288852 21919 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.291566 21920 leader_election.cc:290] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:46.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.292014 21919 leader_election.cc:290] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:43:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:46.296861 21142 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:43:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:46.305337 21386 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:43:46.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:46.311517 21142 leader_election.cc:336] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:43:46.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:46.312646 21386 leader_election.cc:336] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:43:46.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.316781 21493 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c5373a0a37e14c38bc09e40a3616b123" is_pre_election: true
10:43:46.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.317400 21493 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cdd21cb83af541e895b0058bf4e0b6d2 in term 1.
10:43:46.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.318538 21142 leader_election.cc:304] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c5373a0a37e14c38bc09e40a3616b123, cdd21cb83af541e895b0058bf4e0b6d2; no voters: a766ec61926c4e27b86cde01c49ea7ed
10:43:46.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.319519 21919 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 1 FOLLOWER]: Leader pre-election won for term 2
10:43:46.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.319933 21919 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 1 FOLLOWER]: Starting leader election (detected failure of leader a766ec61926c4e27b86cde01c49ea7ed)
10:43:46.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.320328 21919 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 1 FOLLOWER]: Advancing to term 2
10:43:46.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.320161 21259 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "c5373a0a37e14c38bc09e40a3616b123" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" is_pre_election: true
10:43:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.325376 21919 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.326284 21259 raft_consensus.cc:2393] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c5373a0a37e14c38bc09e40a3616b123 in current term 2: Already voted for candidate cdd21cb83af541e895b0058bf4e0b6d2 in this term.
10:43:46.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.327366 21919 leader_election.cc:290] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 2 election: Requested vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:43:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.329020 21493 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:43:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.329596 21493 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 1 FOLLOWER]: Advancing to term 2
10:43:46.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:46.331439 21142 leader_election.cc:336] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:43:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.331996 21386 leader_election.cc:304] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c5373a0a37e14c38bc09e40a3616b123; no voters: a766ec61926c4e27b86cde01c49ea7ed, cdd21cb83af541e895b0058bf4e0b6d2
10:43:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.335059 21493 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cdd21cb83af541e895b0058bf4e0b6d2 in term 2.
10:43:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.335461 21920 raft_consensus.cc:2749] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
10:43:46.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.336158 21142 leader_election.cc:304] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c5373a0a37e14c38bc09e40a3616b123, cdd21cb83af541e895b0058bf4e0b6d2; no voters: a766ec61926c4e27b86cde01c49ea7ed
10:43:46.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.336942 21919 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 2 FOLLOWER]: Leader election won for term 2
10:43:46.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.338527 21919 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 2 LEADER]: Becoming Leader. State: Replica: cdd21cb83af541e895b0058bf4e0b6d2, State: Running, Role: LEADER
10:43:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.339587 21919 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.349668 20699 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 reported cstate change: term changed from 1 to 2, leader changed from a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195) to cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193). New cstate: current_term: 2 leader_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: HEALTHY } } }
10:43:46.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:46.735332 21142 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 -> Peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Couldn't send request to peer a766ec61926c4e27b86cde01c49ea7ed. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:43:46.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.915211 21493 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 2 FOLLOWER]: Refusing update from remote peer cdd21cb83af541e895b0058bf4e0b6d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
10:43:46.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:46.916721 21919 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
10:43:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:49.287876 21142 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 -> Peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Couldn't send request to peer a766ec61926c4e27b86cde01c49ea7ed. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:43:49.965 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:49.966 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:43:49.966 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=5082), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [3ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [27ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [27ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [46ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [46ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [83ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [86ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [86ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [161ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [163ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [165ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [165ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [617ms] refreshing cache from master, [617ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [622ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [623ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [625ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [626ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1018ms] refreshing cache from master, [1018ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [1022ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [1024ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1026ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1026ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [5077ms] refreshing cache from master, [5078ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [5082ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@570558580(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@eba7d4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843) -> (continuation of Deferred@680613589 after retry RPC@413456673) -> (continuation of Deferred@1023605853 after retry RPC@370802624) -> (continuation of Deferred@1238989935 after retry RPC@365226792) -> (continuation of Deferred@1041915802 after retry RPC@213216986) -> (continuation of Deferred@1304699282 after retry RPC@1788888398), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a350d9c -> wakeup thread Time-limited test -> (continuation of Deferred@1872547960 after retry RPC@331530843) -> (continuation of Deferred@680613589 after retry RPC@413456673) -> (continuation of Deferred@1023605853 after retry RPC@370802624) -> (continuation of Deferred@1238989935 after retry RPC@365226792) -> (continuation of Deferred@1041915802 after retry RPC@213216986) -> (continuation of Deferred@1304699282 after retry RPC@1788888398))) after lookup
10:43:49.967 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:43:49.968 [DEBUG - kudu-nio-2] (Connection.java:757) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] Successfully connected to peer
10:43:49.989 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x7c40f732, L:/127.0.0.1:50876 - R:/127.20.36.193:34951] using TOKEN/null
10:43:50.040 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.36.195:44799
10:43:50.041 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

10:43:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:50.043511 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/logs
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.195:44799
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.195
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44795
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.195
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:43:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:50.369854 21990 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:43:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:50.370471 21990 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:43:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:50.371255 21990 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:43:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:50.404882 21990 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:43:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:50.405913 21990 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.195
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:50.441969 21990 tablet_server_runner.cc:78] Tablet server non-default flags:
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.195:44799
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:43:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.195
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44795
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.195
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/logs
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:43:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:43:50.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:50.443789 21990 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:43:50.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:50.445474 21990 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:43:50.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:50.458943 22003 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:50.464318 22012 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:50.460388 22006 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:51.718992 22007 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1255 milliseconds
10:43:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.719108 21990 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:43:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.720477 21990 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:43:51.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.722882 21990 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:43:51.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.724262 21990 hybrid_clock.cc:648] HybridClock initialized: now 1786445031724197 us; error 79 us; skew 500 ppm
10:43:51.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.725099 21990 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:43:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.731266 21990 webserver.cc:533] Webserver started at http://127.20.36.195:44795/ using document root <none> and password file <none>
10:43:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.732224 21990 fs_manager.cc:362] Metadata directory not provided
10:43:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.732440 21990 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:43:51.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.740494 21990 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.002s
10:43:51.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.745133 22026 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:43:51.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.746358 21990 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.004s
10:43:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.746673 21990 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:43:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a766ec61926c4e27b86cde01c49ea7ed"
10:43:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:42 on dist-test-slave-gw5c"
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.748584 21990 fs_report.cc:389] FS layout report
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/data
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:43:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:43:51.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.801714 21990 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:43:51.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.803313 21990 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:43:51.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.803800 21990 kserver.cc:163] Server-wide thread pool size limit: 3276
10:43:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.806483 21990 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:43:51.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.812469 22041 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
10:43:51.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.820963 21990 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
10:43:51.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.821216 21990 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
10:43:51.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.821595 21990 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
10:43:51.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.826443 21990 ts_tablet_manager.cc:616] Registered 1 tablets
10:43:51.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.826655 21990 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
10:43:51.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.826967 22041 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap starting.
10:43:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:51.840876 21142 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111) [suppressed 11 similar messages]
10:43:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:51.845608 21142 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 -> Peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Couldn't send request to peer a766ec61926c4e27b86cde01c49ea7ed. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
10:43:51.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.899458 22041 log.cc:826] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Log is configured to *not* fsync() on all Append() calls
10:43:51.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.943368 22041 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:43:51.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.944599 22041 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap complete.
10:43:51.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.946774 22041 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Time spent bootstrapping tablet: real 0.120s	user 0.087s	sys 0.024s
10:43:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.978263 22041 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.982527 22041 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a766ec61926c4e27b86cde01c49ea7ed, State: Initialized, Role: FOLLOWER
10:43:51.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.983942 22041 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:51.990345 22041 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Time spent starting tablet: real 0.043s	user 0.030s	sys 0.012s
10:43:52.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.023343 21990 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.195:44799
10:43:52.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.023756 22218 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.195:44799 every 8 connection(s)
10:43:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.026187 21990 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:43:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.028501 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 21990
10:43:52.029 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:43:52.068 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:43:52.069 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.36.193:34951
10:43:52.070 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

10:43:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.070515 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 21107
10:43:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.097114 22222 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:43:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.097595 22222 heartbeater.cc:461] Registering TS with master...
10:43:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.097788 22219 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:43:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.098418 22219 heartbeater.cc:461] Registering TS with master...
10:43:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.099015 22222 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:43:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.099677 22219 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:43:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.103554 20698 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:43:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.104157 20948 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:43:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.106844 20698 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.195:60515
10:43:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.110599 22221 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:43:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.110997 22221 heartbeater.cc:461] Registering TS with master...
10:43:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.111738 22221 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:43:52.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:52.115228 20824 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:43:52.123 [DEBUG - kudu-nio-2] (Connection.java:239) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] handling channelInactive
10:43:52.123 [DEBUG - kudu-nio-2] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state READY due to: connection closed
10:43:52.123 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:43:52.124 [DEBUG - Time-limited test] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:43:52.125 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.125 [DEBUG - kudu-nio-3] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.125 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.145 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:52.145 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:43:52.146 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [2ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@608262009(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@7b087d8c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@597af6e5 -> wakeup thread Time-limited test)) after lookup
10:43:52.146 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:43:52.147 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.147 [DEBUG - kudu-nio-4] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.147 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.165 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:52.165 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:43:52.165 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [2ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [24ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@608262009(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@7b087d8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@597af6e5 -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835))) after lookup
10:43:52.166 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:43:52.167 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.167 [DEBUG - kudu-nio-5] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.168 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.204 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:52.204 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:43:52.205 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=81), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [2ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [24ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [44ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [45ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@608262009(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@7b087d8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835) -> (continuation of Deferred@54452771 after retry RPC@558874441), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@597af6e5 -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835) -> (continuation of Deferred@54452771 after retry RPC@558874441))) after lookup
10:43:52.205 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:43:52.207 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.207 [DEBUG - kudu-nio-6] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.207 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.265 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:52.265 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:43:52.266 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=142), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [2ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [24ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [44ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [45ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [82ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [84ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [84ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [141ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@608262009(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@7b087d8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835) -> (continuation of Deferred@54452771 after retry RPC@558874441) -> (continuation of Deferred@1819106899 after retry RPC@813961568), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@597af6e5 -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835) -> (continuation of Deferred@54452771 after retry RPC@558874441) -> (continuation of Deferred@1819106899 after retry RPC@813961568))) after lookup
10:43:52.266 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:43:52.268 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.268 [DEBUG - kudu-nio-7] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.268 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.485 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:52.485 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:43:52.486 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=362), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [2ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [24ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [44ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [45ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [82ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [84ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [84ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [141ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [143ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [145ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [145ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [357ms] refreshing cache from master, [357ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [362ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@608262009(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@7b087d8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835) -> (continuation of Deferred@54452771 after retry RPC@558874441) -> (continuation of Deferred@1819106899 after retry RPC@813961568) -> (continuation of Deferred@1504892310 after retry RPC@486273354), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@597af6e5 -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835) -> (continuation of Deferred@54452771 after retry RPC@558874441) -> (continuation of Deferred@1819106899 after retry RPC@813961568) -> (continuation of Deferred@1504892310 after retry RPC@486273354))) after lookup
10:43:52.487 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:43:52.488 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.488 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:52.488 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:53.045 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:53.046 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:43:53.046 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=922), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [2ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [24ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [44ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [45ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [82ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [84ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [84ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [141ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [143ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [145ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [145ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [357ms] refreshing cache from master, [357ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [362ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [364ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [365ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [365ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [918ms] refreshing cache from master, [918ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [922ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@608262009(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@7b087d8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835) -> (continuation of Deferred@54452771 after retry RPC@558874441) -> (continuation of Deferred@1819106899 after retry RPC@813961568) -> (continuation of Deferred@1504892310 after retry RPC@486273354) -> (continuation of Deferred@1006566704 after retry RPC@2018257572), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@597af6e5 -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835) -> (continuation of Deferred@54452771 after retry RPC@558874441) -> (continuation of Deferred@1819106899 after retry RPC@813961568) -> (continuation of Deferred@1504892310 after retry RPC@486273354) -> (continuation of Deferred@1006566704 after retry RPC@2018257572))) after lookup
10:43:53.047 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:43:53.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:53.048 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:53.048 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:43:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.111519 22222 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:43:53.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.116636 22241 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader cdd21cb83af541e895b0058bf4e0b6d2)
10:43:53.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.117164 22241 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.118911 22241 leader_election.cc:290] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:53.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:53.123559 21386 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111)
10:43:53.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:53.126940 21386 leader_election.cc:336] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111)
10:43:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.130796 22140 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "c5373a0a37e14c38bc09e40a3616b123" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "a766ec61926c4e27b86cde01c49ea7ed" is_pre_election: true
10:43:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.131592 22140 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c5373a0a37e14c38bc09e40a3616b123 in term 1.
10:43:53.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.132853 21386 leader_election.cc:304] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, c5373a0a37e14c38bc09e40a3616b123; no voters: cdd21cb83af541e895b0058bf4e0b6d2
10:43:53.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.133443 22241 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 2 FOLLOWER]: Leader pre-election won for term 3
10:43:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.133785 22241 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 2 FOLLOWER]: Starting leader election (detected failure of leader cdd21cb83af541e895b0058bf4e0b6d2)
10:43:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.134119 22241 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 2 FOLLOWER]: Advancing to term 3
10:43:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.138890 22241 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:53.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.140419 22241 leader_election.cc:290] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 3 election: Requested vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:53.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.141333 22140 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "c5373a0a37e14c38bc09e40a3616b123" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "a766ec61926c4e27b86cde01c49ea7ed"
10:43:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.141906 22140 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 1 FOLLOWER]: Advancing to term 3
10:43:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:53.144464 21386 leader_election.cc:336] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111)
10:43:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.148586 22140 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c5373a0a37e14c38bc09e40a3616b123 in term 3.
10:43:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.149477 21386 leader_election.cc:304] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, c5373a0a37e14c38bc09e40a3616b123; no voters: cdd21cb83af541e895b0058bf4e0b6d2
10:43:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.150148 22241 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 3 FOLLOWER]: Leader election won for term 3
10:43:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.151216 22241 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 3 LEADER]: Becoming Leader. State: Replica: c5373a0a37e14c38bc09e40a3616b123, State: Running, Role: LEADER
10:43:53.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.152212 22241 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.160946 20698 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 reported cstate change: term changed from 2 to 3, leader changed from cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193) to c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194). New cstate: current_term: 3 leader_uuid: "c5373a0a37e14c38bc09e40a3616b123" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: UNKNOWN } } }
10:43:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:53.558266 21386 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 -> Peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Couldn't send request to peer cdd21cb83af541e895b0058bf4e0b6d2. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:43:53.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.587477 22140 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 3 FOLLOWER]: Refusing update from remote peer c5373a0a37e14c38bc09e40a3616b123: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
10:43:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:53.589095 22241 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
10:43:54.805 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:54.805 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:54.806 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=2682), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [2ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [21ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [24ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [44ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [45ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [81ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [82ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [84ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [84ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [141ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [143ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [145ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [145ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [357ms] refreshing cache from master, [357ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [362ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [364ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [365ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [365ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [918ms] refreshing cache from master, [918ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [922ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [924ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [925ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [925ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [2677ms] refreshing cache from master, [2678ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [2682ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@608262009(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@7b087d8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835) -> (continuation of Deferred@54452771 after retry RPC@558874441) -> (continuation of Deferred@1819106899 after retry RPC@813961568) -> (continuation of Deferred@1504892310 after retry RPC@486273354) -> (continuation of Deferred@1006566704 after retry RPC@2018257572) -> (continuation of Deferred@1180427748 after retry RPC@1608248961), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@597af6e5 -> wakeup thread Time-limited test -> (continuation of Deferred@874513340 after retry RPC@987960835) -> (continuation of Deferred@54452771 after retry RPC@558874441) -> (continuation of Deferred@1819106899 after retry RPC@813961568) -> (continuation of Deferred@1504892310 after retry RPC@486273354) -> (continuation of Deferred@1006566704 after retry RPC@2018257572) -> (continuation of Deferred@1180427748 after retry RPC@1608248961))) after lookup
10:43:54.807 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:43:54.808 [DEBUG - kudu-nio-2] (Connection.java:757) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] Successfully connected to peer
10:43:54.834 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xd2b47538, L:/127.0.0.1:45756 - R:/127.20.36.194:39729] using TOKEN/null
10:43:54.886 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.36.193:34951
10:43:54.887 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:54.888837 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/logs
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.193:34951
10:43:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.193
10:43:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35283
10:43:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.193
10:43:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:43:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:55.239534 22292 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:43:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:55.240144 22292 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:43:55.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:55.240939 22292 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:43:55.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:55.274977 22292 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:43:55.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:55.276165 22292 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.193
10:43:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:55.313184 22292 tablet_server_runner.cc:78] Tablet server non-default flags:
10:43:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:43:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:43:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:43:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:43:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data
10:43:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.193:34951
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.193
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35283
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.193
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/logs
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:43:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:55.314471 22292 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:43:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:55.316277 22292 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:43:55.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:55.328163 22306 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:56.148808 21386 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 -> Peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Couldn't send request to peer cdd21cb83af541e895b0058bf4e0b6d2. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:43:57.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:56.732954 22303 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 22292
10:43:57.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:55.330160 22308 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:57.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:57.049176 22303 kernel_stack_watchdog.cc:198] Thread 22292 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 400ms:
10:43:57.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
10:43:57.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
10:43:57.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
10:43:57.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
10:43:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:57.054872 22292 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.725s	user 0.001s	sys 0.001s
10:43:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:57.054926 22309 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1720 milliseconds
10:43:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:57.055305 22292 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.726s	user 0.001s	sys 0.001s
10:43:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:57.056864 22310 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:43:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.056883 22292 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:43:57.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.058537 22292 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:43:57.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.061286 22292 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:43:57.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.062738 22292 hybrid_clock.cc:648] HybridClock initialized: now 1786445037062612 us; error 117 us; skew 500 ppm
10:43:57.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.063792 22292 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:43:57.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.071425 22292 webserver.cc:533] Webserver started at http://127.20.36.193:35283/ using document root <none> and password file <none>
10:43:57.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.072650 22292 fs_manager.cc:362] Metadata directory not provided
10:43:57.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.072955 22292 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:43:57.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.081893 22292 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
10:43:57.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.086460 22329 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:43:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.087666 22292 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
10:43:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.087980 22292 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdd21cb83af541e895b0058bf4e0b6d2"
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:38 on dist-test-slave-gw5c"
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.089941 22292 fs_report.cc:389] FS layout report
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/data
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:43:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:43:57.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.160935 22292 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:43:57.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.162467 22292 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:43:57.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.162987 22292 kserver.cc:163] Server-wide thread pool size limit: 3276
10:43:57.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.165946 22292 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:43:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.171754 22336 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
10:43:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.180744 22292 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
10:43:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.180985 22292 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.002s
10:43:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.181205 22292 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
10:43:57.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.185914 22292 ts_tablet_manager.cc:616] Registered 1 tablets
10:43:57.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.186151 22292 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.001s
10:43:57.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.186539 22336 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Bootstrap starting.
10:43:57.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.245003 22336 log.cc:826] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Log is configured to *not* fsync() on all Append() calls
10:43:57.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.291158 22336 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:43:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.292357 22336 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Bootstrap complete.
10:43:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.295250 22336 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Time spent bootstrapping tablet: real 0.109s	user 0.062s	sys 0.042s
10:43:57.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.315196 22336 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:57.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.318053 22336 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdd21cb83af541e895b0058bf4e0b6d2, State: Initialized, Role: FOLLOWER
10:43:57.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.319481 22336 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.323794 22336 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Time spent starting tablet: real 0.028s	user 0.022s	sys 0.006s
10:43:57.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.403535 22292 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.193:34951
10:43:57.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.403796 22516 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.193:34951 every 8 connection(s)
10:43:57.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.406216 22292 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:43:57.409 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:43:57.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.408628 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 22292
10:43:57.432 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:43:57.432 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.36.194:39729
10:43:57.433 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

10:43:57.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.433696 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 21354
10:43:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.466372 22517 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:43:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.466923 22517 heartbeater.cc:461] Registering TS with master...
10:43:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.468410 22517 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:43:57.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.473238 20951 ts_manager.cc:194] Re-registered known tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:57.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.473951 22519 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:43:57.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.474261 22519 heartbeater.cc:461] Registering TS with master...
10:43:57.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.475204 22519 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:43:57.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.478302 20699 ts_manager.cc:194] Re-registered known tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:57.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.480415 22518 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:43:57.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.480746 22518 heartbeater.cc:461] Registering TS with master...
10:43:57.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.481181 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.193:39191
10:43:57.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.481738 22518 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:43:57.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:57.485368 20824 ts_manager.cc:194] Re-registered known tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:57.497 [DEBUG - kudu-nio-2] (Connection.java:239) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] handling channelInactive
10:43:57.497 [DEBUG - kudu-nio-2] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state READY due to: connection closed
10:43:57.497 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:43:57.498 [DEBUG - Time-limited test] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:43:57.499 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.499 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.499 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.524 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:57.525 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:57.525 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [1ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [22ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@691614998(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@5a018f70 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a4f1cbc -> wakeup thread Time-limited test)) after lookup
10:43:57.526 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:43:57.527 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.527 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.527 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.544 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:57.544 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:57.544 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [1ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [22ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [28ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [29ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [29ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@691614998(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@5a018f70 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a4f1cbc -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100))) after lookup
10:43:57.545 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:43:57.546 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.546 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.546 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.565 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:57.565 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:57.566 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=68), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [1ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [22ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [28ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [29ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [29ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [47ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [48ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [48ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [67ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@691614998(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@5a018f70 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100) -> (continuation of Deferred@639067485 after retry RPC@340961341), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a4f1cbc -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100) -> (continuation of Deferred@639067485 after retry RPC@340961341))) after lookup
10:43:57.567 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:43:57.568 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.568 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.568 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.645 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:57.645 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:57.645 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=147), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [1ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [22ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [28ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [29ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [29ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [47ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [48ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [48ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [67ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [68ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [70ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [70ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [142ms] refreshing cache from master, [143ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [146ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@691614998(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@5a018f70 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100) -> (continuation of Deferred@639067485 after retry RPC@340961341) -> (continuation of Deferred@21989495 after retry RPC@6146851), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a4f1cbc -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100) -> (continuation of Deferred@639067485 after retry RPC@340961341) -> (continuation of Deferred@21989495 after retry RPC@6146851))) after lookup
10:43:57.646 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:43:57.647 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.647 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.647 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.825 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:57.825 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:57.825 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=327), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [1ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [22ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [28ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [29ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [29ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [47ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [48ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [48ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [67ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [68ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [70ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [70ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [142ms] refreshing cache from master, [143ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [146ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [147ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [149ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [149ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [322ms] refreshing cache from master, [323ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [326ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@691614998(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@5a018f70 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100) -> (continuation of Deferred@639067485 after retry RPC@340961341) -> (continuation of Deferred@21989495 after retry RPC@6146851) -> (continuation of Deferred@1731105037 after retry RPC@155474456), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a4f1cbc -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100) -> (continuation of Deferred@639067485 after retry RPC@340961341) -> (continuation of Deferred@21989495 after retry RPC@6146851) -> (continuation of Deferred@1731105037 after retry RPC@155474456))) after lookup
10:43:57.826 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:43:57.827 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.827 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:57.827 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.484207 22519 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:43:58.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.799541 22541 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 3 FOLLOWER]: Starting pre-election (detected failure of leader c5373a0a37e14c38bc09e40a3616b123)
10:43:58.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.799949 22541 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:58.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.803056 22541 leader_election.cc:290] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:58.806 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:43:58.806 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:43:58.807 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1309), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [1ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [22ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [28ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [29ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [29ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [47ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [48ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [48ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [67ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [68ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [70ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [70ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [142ms] refreshing cache from master, [143ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [146ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [147ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [149ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [149ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [322ms] refreshing cache from master, [323ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [326ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [328ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [329ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [329ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [1302ms] refreshing cache from master, [1303ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [1308ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@691614998(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@5a018f70 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100) -> (continuation of Deferred@639067485 after retry RPC@340961341) -> (continuation of Deferred@21989495 after retry RPC@6146851) -> (continuation of Deferred@1731105037 after retry RPC@155474456) -> (continuation of Deferred@1632114911 after retry RPC@1249315444), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a4f1cbc -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100) -> (continuation of Deferred@639067485 after retry RPC@340961341) -> (continuation of Deferred@21989495 after retry RPC@6146851) -> (continuation of Deferred@1731105037 after retry RPC@155474456) -> (continuation of Deferred@1632114911 after retry RPC@1249315444))) after lookup
10:43:58.808 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:43:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:58.810381 22029 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111)
10:43:58.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:58.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:58.811 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:43:58.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:58.815356 22029 leader_election.cc:336] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111)
10:43:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.820466 22451 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "a766ec61926c4e27b86cde01c49ea7ed" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" is_pre_election: true
10:43:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.821234 22451 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a766ec61926c4e27b86cde01c49ea7ed in term 2.
10:43:58.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.822609 22029 leader_election.cc:304] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, cdd21cb83af541e895b0058bf4e0b6d2; no voters: c5373a0a37e14c38bc09e40a3616b123
10:43:58.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.823233 22541 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 3 FOLLOWER]: Leader pre-election won for term 4
10:43:58.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.823591 22541 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 3 FOLLOWER]: Starting leader election (detected failure of leader c5373a0a37e14c38bc09e40a3616b123)
10:43:58.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.823936 22541 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 3 FOLLOWER]: Advancing to term 4
10:43:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.828550 22541 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:58.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.830008 22541 leader_election.cc:290] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 4 election: Requested vote from peers c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:43:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.831650 22451 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "a766ec61926c4e27b86cde01c49ea7ed" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "cdd21cb83af541e895b0058bf4e0b6d2"
10:43:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.832214 22451 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 2 FOLLOWER]: Advancing to term 4
10:43:58.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:58.833788 22029 leader_election.cc:336] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111)
10:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.838807 22451 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a766ec61926c4e27b86cde01c49ea7ed in term 4.
10:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.839795 22029 leader_election.cc:304] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, cdd21cb83af541e895b0058bf4e0b6d2; no voters: c5373a0a37e14c38bc09e40a3616b123
10:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.840377 22541 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 4 FOLLOWER]: Leader election won for term 4
10:43:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.841883 22541 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 4 LEADER]: Becoming Leader. State: Replica: a766ec61926c4e27b86cde01c49ea7ed, State: Running, Role: LEADER
10:43:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.842792 22541 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:58.851540 20699 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed reported cstate change: term changed from 3 to 4, leader changed from c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194) to a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195). New cstate: current_term: 4 leader_uuid: "a766ec61926c4e27b86cde01c49ea7ed" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: UNKNOWN } } }
10:43:58.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:58.935374 20873 debug-util.cc:398] Leaking SignalData structure 0x7b080009a4a0 after lost signal to thread 20772
10:43:58.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:58.936128 20873 debug-util.cc:398] Leaking SignalData structure 0x7b08000912e0 after lost signal to thread 20879
10:43:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:59.377424 22451 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 4 FOLLOWER]: Refusing update from remote peer a766ec61926c4e27b86cde01c49ea7ed: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
10:43:59.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:43:59.379633 22570 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [LEADER]: Connected to new peer: Peer: permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
10:43:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:43:59.473929 22029 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed -> Peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Couldn't send request to peer c5373a0a37e14c38bc09e40a3616b123. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:44:01.324 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:01.325 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:01.326 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3828), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [1ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [22ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [28ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [29ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [29ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [47ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [48ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [48ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [67ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [68ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [70ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [70ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [142ms] refreshing cache from master, [143ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [146ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [147ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [149ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [149ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [322ms] refreshing cache from master, [323ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [326ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [328ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [329ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [329ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [1302ms] refreshing cache from master, [1303ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [1308ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [1310ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [1313ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [1313ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [3822ms] refreshing cache from master, [3822ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [3826ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@691614998(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@5a018f70 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100) -> (continuation of Deferred@639067485 after retry RPC@340961341) -> (continuation of Deferred@21989495 after retry RPC@6146851) -> (continuation of Deferred@1731105037 after retry RPC@155474456) -> (continuation of Deferred@1632114911 after retry RPC@1249315444) -> (continuation of Deferred@1349814342 after retry RPC@1742751923), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6a4f1cbc -> wakeup thread Time-limited test -> (continuation of Deferred@1052483339 after retry RPC@738451100) -> (continuation of Deferred@639067485 after retry RPC@340961341) -> (continuation of Deferred@21989495 after retry RPC@6146851) -> (continuation of Deferred@1731105037 after retry RPC@155474456) -> (continuation of Deferred@1632114911 after retry RPC@1249315444) -> (continuation of Deferred@1349814342 after retry RPC@1742751923))) after lookup
10:44:01.327 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:01.328 [DEBUG - kudu-nio-2] (Connection.java:757) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] Successfully connected to peer
10:44:01.351 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xbb360ab8, L:/127.0.0.1:45454 - R:/127.20.36.195:44799] using TOKEN/null
10:44:01.409 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.36.194:39729
10:44:01.410 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:01.411835 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/logs
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.194:39729
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.194
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45775
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.194
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:44:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:01.751700 22610 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:44:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:01.752180 22610 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:44:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:01.752640 22610 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:44:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:01.785176 22610 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:44:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:01.786024 22610 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.194
10:44:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:01.823251 22610 tablet_server_runner.cc:78] Tablet server non-default flags:
10:44:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.194:39729
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.194
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45775
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.194
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/logs
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:44:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:01.824651 22610 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:44:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:01.826243 22610 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:44:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:01.840333 22626 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:01.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:01.918147 22029 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed -> Peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Couldn't send request to peer c5373a0a37e14c38bc09e40a3616b123. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:44:03.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:01.841063 22627 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:03.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:03.244555 22624 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 22610
10:44:03.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:03.257611 22610 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.417s	user 0.447s	sys 0.943s
10:44:03.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:03.259569 22610 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.419s	user 0.447s	sys 0.944s
10:44:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:03.260357 22632 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:03.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:03.265002 22628 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1416 milliseconds
10:44:03.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.265007 22610 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:44:03.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.266557 22610 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:44:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.268865 22610 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:44:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.270279 22610 hybrid_clock.cc:648] HybridClock initialized: now 1786445043270208 us; error 68 us; skew 500 ppm
10:44:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.271166 22610 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:44:03.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.278431 22610 webserver.cc:533] Webserver started at http://127.20.36.194:45775/ using document root <none> and password file <none>
10:44:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.279727 22610 fs_manager.cc:362] Metadata directory not provided
10:44:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.280051 22610 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:44:03.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.289674 22610 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.004s	sys 0.001s
10:44:03.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.294317 22654 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:44:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.295634 22610 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.295995 22610 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:40 on dist-test-slave-gw5c"
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.297945 22610 fs_report.cc:389] FS layout report
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/data
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:44:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:44:03.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.360575 22610 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:44:03.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.362542 22610 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:44:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.363073 22610 kserver.cc:163] Server-wide thread pool size limit: 3276
10:44:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.365849 22610 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:44:03.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.372308 22664 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
10:44:03.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.385003 22610 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
10:44:03.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.385262 22610 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.015s	user 0.000s	sys 0.002s
10:44:03.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.385493 22610 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
10:44:03.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.390475 22610 ts_tablet_manager.cc:616] Registered 1 tablets
10:44:03.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.390677 22610 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
10:44:03.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.391218 22664 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Bootstrap starting.
10:44:03.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.458889 22664 log.cc:826] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Log is configured to *not* fsync() on all Append() calls
10:44:03.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.514370 22664 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:44:03.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.515368 22664 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Bootstrap complete.
10:44:03.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.516978 22664 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Time spent bootstrapping tablet: real 0.126s	user 0.092s	sys 0.029s
10:44:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.538012 22664 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.540983 22664 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5373a0a37e14c38bc09e40a3616b123, State: Initialized, Role: FOLLOWER
10:44:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.542105 22664 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.546312 22664 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Time spent starting tablet: real 0.029s	user 0.023s	sys 0.007s
10:44:03.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.594692 22610 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.194:39729
10:44:03.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.594993 22838 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.194:39729 every 8 connection(s)
10:44:03.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.597270 22610 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:44:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.604282 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 22610
10:44:03.605 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:03.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.642175 22840 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:44:03.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.642663 22840 heartbeater.cc:461] Registering TS with master...
10:44:03.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.644052 22840 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:44:03.650 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:44:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.657518 22842 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:44:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.657935 22842 heartbeater.cc:461] Registering TS with master...
10:44:03.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.659112 22842 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:44:03.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.660403 22841 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:44:03.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.660777 22841 heartbeater.cc:461] Registering TS with master...
10:44:03.661 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.36.195:44799
10:44:03.661 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

10:44:03.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.662110 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 21990
10:44:03.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.662348 22841 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:44:03.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.677856 20699 ts_manager.cc:194] Re-registered known tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:03.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.678772 20951 ts_manager.cc:194] Re-registered known tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:03.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.681452 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.194:53647
10:44:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:03.686149 20824 ts_manager.cc:194] Re-registered known tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:03.715 [DEBUG - kudu-nio-2] (Connection.java:239) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] handling channelInactive
10:44:03.715 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state READY due to: connection closed
10:44:03.716 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:03.717 [DEBUG - Time-limited test] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:03.718 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.718 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.719 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.725 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:03.725 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:03.725 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=8), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [7ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@678472640(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@3dca897f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38684bb3 -> wakeup thread Time-limited test)) after lookup
10:44:03.726 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:03.727 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.727 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.727 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.743 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:03.744 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:03.744 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [7ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [8ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [10ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [10ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@678472640(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@3dca897f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@744702106 after retry RPC@1445770102), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38684bb3 -> wakeup thread Time-limited test -> (continuation of Deferred@744702106 after retry RPC@1445770102))) after lookup
10:44:03.744 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:03.745 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.745 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.745 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.764 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:03.765 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:03.765 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [7ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [8ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [10ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [10ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [27ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [28ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [28ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [47ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@678472640(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@3dca897f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@744702106 after retry RPC@1445770102) -> (continuation of Deferred@1359762833 after retry RPC@1644755909), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38684bb3 -> wakeup thread Time-limited test -> (continuation of Deferred@744702106 after retry RPC@1445770102) -> (continuation of Deferred@1359762833 after retry RPC@1644755909))) after lookup
10:44:03.766 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:03.767 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.767 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.767 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.845 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:03.845 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:03.845 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=128), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [7ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [8ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [10ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [10ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [27ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [28ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [28ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [47ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [48ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [50ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [50ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [127ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@678472640(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@3dca897f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@744702106 after retry RPC@1445770102) -> (continuation of Deferred@1359762833 after retry RPC@1644755909) -> (continuation of Deferred@1609110205 after retry RPC@2048531542), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38684bb3 -> wakeup thread Time-limited test -> (continuation of Deferred@744702106 after retry RPC@1445770102) -> (continuation of Deferred@1359762833 after retry RPC@1644755909) -> (continuation of Deferred@1609110205 after retry RPC@2048531542))) after lookup
10:44:03.846 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:03.847 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.847 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:03.848 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:04.344 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:04.344 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:04.344 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=627), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [7ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [8ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [10ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [10ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [27ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [28ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [28ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [47ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [48ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [50ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [50ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [127ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [129ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [130ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [131ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [623ms] refreshing cache from master, [623ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [626ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@678472640(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@3dca897f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@744702106 after retry RPC@1445770102) -> (continuation of Deferred@1359762833 after retry RPC@1644755909) -> (continuation of Deferred@1609110205 after retry RPC@2048531542) -> (continuation of Deferred@1201631753 after retry RPC@1842167296), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38684bb3 -> wakeup thread Time-limited test -> (continuation of Deferred@744702106 after retry RPC@1445770102) -> (continuation of Deferred@1359762833 after retry RPC@1644755909) -> (continuation of Deferred@1609110205 after retry RPC@2048531542) -> (continuation of Deferred@1201631753 after retry RPC@1842167296))) after lookup
10:44:04.345 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:04.346 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:04.346 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:04.346 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.688252 22842 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:44:04.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.783344 22865 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader a766ec61926c4e27b86cde01c49ea7ed)
10:44:04.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.783883 22865 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:04.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.786463 22865 leader_election.cc:290] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:04.789652 22330 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:04.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:04.793320 22330 leader_election.cc:336] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.799190 22767 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "c5373a0a37e14c38bc09e40a3616b123" is_pre_election: true
10:44:04.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.799974 22767 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cdd21cb83af541e895b0058bf4e0b6d2 in term 3.
10:44:04.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.801131 22330 leader_election.cc:304] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c5373a0a37e14c38bc09e40a3616b123, cdd21cb83af541e895b0058bf4e0b6d2; no voters: a766ec61926c4e27b86cde01c49ea7ed
10:44:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.801751 22865 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 4 FOLLOWER]: Leader pre-election won for term 5
10:44:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.802069 22865 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 4 FOLLOWER]: Starting leader election (detected failure of leader a766ec61926c4e27b86cde01c49ea7ed)
10:44:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.802297 22865 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 4 FOLLOWER]: Advancing to term 5
10:44:04.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.806638 22865 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:04.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.808295 22865 leader_election.cc:290] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 5 election: Requested vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:04.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.809870 22767 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:44:04.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.810495 22767 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 3 FOLLOWER]: Advancing to term 5
10:44:04.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:04.811535 22330 leader_election.cc:336] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:04.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.817060 22767 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cdd21cb83af541e895b0058bf4e0b6d2 in term 5.
10:44:04.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.818030 22330 leader_election.cc:304] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c5373a0a37e14c38bc09e40a3616b123, cdd21cb83af541e895b0058bf4e0b6d2; no voters: a766ec61926c4e27b86cde01c49ea7ed
10:44:04.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.818722 22865 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 5 FOLLOWER]: Leader election won for term 5
10:44:04.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.820429 22865 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 5 LEADER]: Becoming Leader. State: Replica: cdd21cb83af541e895b0058bf4e0b6d2, State: Running, Role: LEADER
10:44:04.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.821337 22865 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:04.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.829743 20699 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 reported cstate change: term changed from 4 to 5, leader changed from a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195) to cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193). New cstate: current_term: 5 leader_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: HEALTHY } } }
10:44:04.945 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:04.945 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:04.946 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1229), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [7ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [8ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [10ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [10ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [26ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [27ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [28ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [28ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [47ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [48ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [50ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [50ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [127ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [129ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [130ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [131ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [623ms] refreshing cache from master, [623ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [626ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [628ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [629ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [629ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1223ms] refreshing cache from master, [1223ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [1227ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@678472640(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@3dca897f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@744702106 after retry RPC@1445770102) -> (continuation of Deferred@1359762833 after retry RPC@1644755909) -> (continuation of Deferred@1609110205 after retry RPC@2048531542) -> (continuation of Deferred@1201631753 after retry RPC@1842167296) -> (continuation of Deferred@1569741914 after retry RPC@1504265430), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38684bb3 -> wakeup thread Time-limited test -> (continuation of Deferred@744702106 after retry RPC@1445770102) -> (continuation of Deferred@1359762833 after retry RPC@1644755909) -> (continuation of Deferred@1609110205 after retry RPC@2048531542) -> (continuation of Deferred@1201631753 after retry RPC@1842167296) -> (continuation of Deferred@1569741914 after retry RPC@1504265430))) after lookup
10:44:04.947 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:04.948 [DEBUG - kudu-nio-1] (Connection.java:757) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] Successfully connected to peer
10:44:04.969 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x567faf0d, L:/127.0.0.1:44578 - R:/127.20.36.193:34951] using TOKEN/null
10:44:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.979537 22767 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 5 FOLLOWER]: Refusing update from remote peer cdd21cb83af541e895b0058bf4e0b6d2: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 18. (index mismatch)
10:44:04.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:04.981379 22865 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
10:44:04.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:04.982095 22330 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 -> Peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Couldn't send request to peer a766ec61926c4e27b86cde01c49ea7ed. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:44:05.051 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.36.195:44799
10:44:05.052 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

10:44:05.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:05.053246 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/logs
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.195:44799
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.195
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44795
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.195
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:05.399973 22910 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:05.400473 22910 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:44:05.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:05.400929 22910 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:44:05.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:05.435966 22910 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:44:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:05.436916 22910 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.195
10:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:05.475210 22910 tablet_server_runner.cc:78] Tablet server non-default flags:
10:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data
10:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.195:44799
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.195
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44795
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.195
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/logs
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:05.476579 22910 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:44:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:05.478295 22910 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:44:05.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:05.492871 22919 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:07.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:06.894338 22918 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 22910
10:44:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:07.273286 22918 kernel_stack_watchdog.cc:198] Thread 22910 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 398ms:
10:44:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
10:44:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
10:44:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
10:44:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
10:44:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:05.493786 22920 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:07.275215 22910 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.783s	user 0.568s	sys 1.188s
10:44:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:07.275836 22910 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.783s	user 0.568s	sys 1.189s
10:44:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:07.276054 22923 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:07.280133 22921 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1782 milliseconds
10:44:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.280181 22910 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:44:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.281533 22910 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:44:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.284257 22910 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:44:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.285823 22910 hybrid_clock.cc:648] HybridClock initialized: now 1786445047285701 us; error 131 us; skew 500 ppm
10:44:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.286710 22910 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:44:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.293349 22910 webserver.cc:533] Webserver started at http://127.20.36.195:44795/ using document root <none> and password file <none>
10:44:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.294358 22910 fs_manager.cc:362] Metadata directory not provided
10:44:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.294622 22910 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:44:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.303071 22910 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
10:44:07.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.308396 22941 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:44:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.309823 22910 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.004s	sys 0.000s
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.310233 22910 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a766ec61926c4e27b86cde01c49ea7ed"
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:42 on dist-test-slave-gw5c"
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:07.312723 22330 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 -> Peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Couldn't send request to peer a766ec61926c4e27b86cde01c49ea7ed. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.312945 22910 fs_report.cc:389] FS layout report
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/data
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:44:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:44:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.364391 22910 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:44:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.365924 22910 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:44:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.366439 22910 kserver.cc:163] Server-wide thread pool size limit: 3276
10:44:07.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.369141 22910 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:44:07.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.377534 22955 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
10:44:07.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.386797 22910 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
10:44:07.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.387084 22910 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.012s	user 0.002s	sys 0.000s
10:44:07.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.387300 22910 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
10:44:07.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.392117 22910 ts_tablet_manager.cc:616] Registered 1 tablets
10:44:07.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.392367 22910 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
10:44:07.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.392810 22955 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap starting.
10:44:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.465111 22955 log.cc:826] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Log is configured to *not* fsync() on all Append() calls
10:44:07.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.550058 22955 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:44:07.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.551618 22955 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap complete.
10:44:07.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.553779 22955 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Time spent bootstrapping tablet: real 0.161s	user 0.092s	sys 0.063s
10:44:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.568872 22910 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.195:44799
10:44:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.568946 23125 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.195:44799 every 8 connection(s)
10:44:07.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.572618 22910 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:44:07.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.582219 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 22910
10:44:07.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:07.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.583526 22955 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:07.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.588013 22955 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: a766ec61926c4e27b86cde01c49ea7ed, State: Initialized, Role: FOLLOWER
10:44:07.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.589287 22955 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:07.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.624508 22955 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Time spent starting tablet: real 0.070s	user 0.036s	sys 0.006s
10:44:07.636 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:44:07.637 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.36.193:34951
10:44:07.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

10:44:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.638715 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 22292
10:44:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.663332 23127 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:44:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.663868 23127 heartbeater.cc:461] Registering TS with master...
10:44:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.663990 23129 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:44:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.664327 23129 heartbeater.cc:461] Registering TS with master...
10:44:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.665405 23127 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:44:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.665580 23129 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:44:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.669672 20699 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:44:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.671717 23128 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:44:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.672130 23128 heartbeater.cc:461] Registering TS with master...
10:44:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.672705 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.195:44885
10:44:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.673095 23128 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:44:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.675403 20951 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:44:07.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.677573 20824 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:44:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:07.679301 23129 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:44:07.691 [DEBUG - kudu-nio-1] (Connection.java:239) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] handling channelInactive
10:44:07.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state READY due to: connection closed
10:44:07.692 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:07.693 [DEBUG - Time-limited test] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:07.694 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.694 [DEBUG - kudu-nio-2] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.694 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.704 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:07.704 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:07.704 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [1ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@327175382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@bae284b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@234a9f11 -> wakeup thread Time-limited test)) after lookup
10:44:07.705 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:07.706 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.706 [DEBUG - kudu-nio-3] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.706 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.724 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:07.724 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:07.724 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [1ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [12ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [13ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [13ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@327175382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@bae284b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@277102253 after retry RPC@1732352443), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@234a9f11 -> wakeup thread Time-limited test -> (continuation of Deferred@277102253 after retry RPC@1732352443))) after lookup
10:44:07.725 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:07.726 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.726 [DEBUG - kudu-nio-4] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.726 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.744 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:07.745 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:07.745 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=52), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [1ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [12ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [13ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [13ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [31ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [33ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [33ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [51ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@327175382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@bae284b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@277102253 after retry RPC@1732352443) -> (continuation of Deferred@2138395961 after retry RPC@345147181), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@234a9f11 -> wakeup thread Time-limited test -> (continuation of Deferred@277102253 after retry RPC@1732352443) -> (continuation of Deferred@2138395961 after retry RPC@345147181))) after lookup
10:44:07.746 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:07.747 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.747 [DEBUG - kudu-nio-5] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.748 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.884 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:07.884 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:07.885 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=192), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [1ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [12ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [13ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [13ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [31ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [33ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [33ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [51ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [52ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [55ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [55ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [191ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@327175382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@bae284b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@277102253 after retry RPC@1732352443) -> (continuation of Deferred@2138395961 after retry RPC@345147181) -> (continuation of Deferred@997497518 after retry RPC@1603383631), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@234a9f11 -> wakeup thread Time-limited test -> (continuation of Deferred@277102253 after retry RPC@1732352443) -> (continuation of Deferred@2138395961 after retry RPC@345147181) -> (continuation of Deferred@997497518 after retry RPC@1603383631))) after lookup
10:44:07.885 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:07.886 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.887 [DEBUG - kudu-nio-6] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:07.887 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:08.045 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:08.045 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:08.045 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=352), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [1ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [12ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [13ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [13ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [31ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [33ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [33ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [51ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [52ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [55ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [55ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [191ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [192ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [194ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [194ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [348ms] refreshing cache from master, [348ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [351ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@327175382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@bae284b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@277102253 after retry RPC@1732352443) -> (continuation of Deferred@2138395961 after retry RPC@345147181) -> (continuation of Deferred@997497518 after retry RPC@1603383631) -> (continuation of Deferred@126825095 after retry RPC@1430708464), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@234a9f11 -> wakeup thread Time-limited test -> (continuation of Deferred@277102253 after retry RPC@1732352443) -> (continuation of Deferred@2138395961 after retry RPC@345147181) -> (continuation of Deferred@997497518 after retry RPC@1603383631) -> (continuation of Deferred@126825095 after retry RPC@1430708464))) after lookup
10:44:08.046 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:08.047 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:08.047 [DEBUG - kudu-nio-7] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:08.047 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:08.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.727635 23150 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader cdd21cb83af541e895b0058bf4e0b6d2)
10:44:08.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.728036 23150 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.730173 23150 leader_election.cc:290] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:08.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:08.734599 22655 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111)
10:44:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:08.737835 22655 leader_election.cc:336] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111)
10:44:08.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.740980 23064 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "c5373a0a37e14c38bc09e40a3616b123" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "a766ec61926c4e27b86cde01c49ea7ed" is_pre_election: true
10:44:08.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.741719 23064 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c5373a0a37e14c38bc09e40a3616b123 in term 4.
10:44:08.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.743031 22655 leader_election.cc:304] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, c5373a0a37e14c38bc09e40a3616b123; no voters: cdd21cb83af541e895b0058bf4e0b6d2
10:44:08.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.743747 23150 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 5 FOLLOWER]: Leader pre-election won for term 6
10:44:08.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.743992 23150 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 5 FOLLOWER]: Starting leader election (detected failure of leader cdd21cb83af541e895b0058bf4e0b6d2)
10:44:08.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.744220 23150 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 5 FOLLOWER]: Advancing to term 6
10:44:08.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.748499 23150 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.749996 23150 leader_election.cc:290] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 6 election: Requested vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.750865 23064 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "c5373a0a37e14c38bc09e40a3616b123" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "a766ec61926c4e27b86cde01c49ea7ed"
10:44:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.751248 23064 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 4 FOLLOWER]: Advancing to term 6
10:44:08.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:08.754489 22655 leader_election.cc:336] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111)
10:44:08.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.757791 23064 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c5373a0a37e14c38bc09e40a3616b123 in term 6.
10:44:08.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.758733 22655 leader_election.cc:304] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, c5373a0a37e14c38bc09e40a3616b123; no voters: cdd21cb83af541e895b0058bf4e0b6d2
10:44:08.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.759418 23150 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 6 FOLLOWER]: Leader election won for term 6
10:44:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.760982 23150 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 6 LEADER]: Becoming Leader. State: Replica: c5373a0a37e14c38bc09e40a3616b123, State: Running, Role: LEADER
10:44:08.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.761821 23150 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:08.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:08.771765 20699 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 reported cstate change: term changed from 5 to 6, leader changed from cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193) to c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194). New cstate: current_term: 6 leader_uuid: "c5373a0a37e14c38bc09e40a3616b123" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: UNKNOWN } } }
10:44:08.984 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:08.985 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:08.985 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1292), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [1ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [12ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [13ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [13ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [30ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [31ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [33ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [33ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [51ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [52ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [55ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [55ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [191ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [192ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [194ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [194ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [348ms] refreshing cache from master, [348ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [351ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [353ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [354ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [354ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [1287ms] refreshing cache from master, [1288ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [1291ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@327175382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@bae284b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@277102253 after retry RPC@1732352443) -> (continuation of Deferred@2138395961 after retry RPC@345147181) -> (continuation of Deferred@997497518 after retry RPC@1603383631) -> (continuation of Deferred@126825095 after retry RPC@1430708464) -> (continuation of Deferred@1592579407 after retry RPC@1011871490), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@234a9f11 -> wakeup thread Time-limited test -> (continuation of Deferred@277102253 after retry RPC@1732352443) -> (continuation of Deferred@2138395961 after retry RPC@345147181) -> (continuation of Deferred@997497518 after retry RPC@1603383631) -> (continuation of Deferred@126825095 after retry RPC@1430708464) -> (continuation of Deferred@1592579407 after retry RPC@1011871490))) after lookup
10:44:08.986 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:08.987 [DEBUG - kudu-nio-0] (Connection.java:757) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] Successfully connected to peer
10:44:09.006 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x744053c8, L:/127.0.0.1:34816 - R:/127.20.36.194:39729] using TOKEN/null
10:44:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:09.016775 23064 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 6 FOLLOWER]: Refusing update from remote peer c5373a0a37e14c38bc09e40a3616b123: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 22. (index mismatch)
10:44:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:09.018047 22655 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 -> Peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Couldn't send request to peer cdd21cb83af541e895b0058bf4e0b6d2. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:44:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:09.019054 23160 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
10:44:09.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:09.044942 23189 mvcc.cc:204] Tried to move back new op lower bound from 7317278920755740672 to 7317278919736541184. Current Snapshot: MvccSnapshot[applied={T|T < 7317278920755740672}]
10:44:09.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:09.053721 23191 mvcc.cc:204] Tried to move back new op lower bound from 7317278920755740672 to 7317278919736541184. Current Snapshot: MvccSnapshot[applied={T|T < 7317278920755740672}]
10:44:09.091 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.36.193:34951
10:44:09.093 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

10:44:09.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:09.095104 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:09.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:09.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:44:09.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data
10:44:09.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:44:09.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:44:09.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:44:09.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:09.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/logs
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.193:34951
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.193
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35283
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.193
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:44:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:09.418874 23199 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:44:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:09.419564 23199 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:44:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:09.420192 23199 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:44:09.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:09.453094 23199 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:44:09.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:09.454121 23199 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.193
10:44:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:09.491206 23199 tablet_server_runner.cc:78] Tablet server non-default flags:
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.193:34951
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.193
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35283
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.193
10:44:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/logs
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:44:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:09.493209 23199 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:44:09.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:09.494957 23199 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:44:09.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:09.510285 23206 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:10.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:09.512987 23207 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:10.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:09.514854 23209 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:10.853658 23208 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1337 milliseconds
10:44:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.853776 23199 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:44:10.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.855075 23199 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:44:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.857339 23199 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:44:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.858697 23199 hybrid_clock.cc:648] HybridClock initialized: now 1786445050858675 us; error 67 us; skew 500 ppm
10:44:10.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.859556 23199 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:44:10.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.865804 23199 webserver.cc:533] Webserver started at http://127.20.36.193:35283/ using document root <none> and password file <none>
10:44:10.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.866690 23199 fs_manager.cc:362] Metadata directory not provided
10:44:10.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.866882 23199 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:44:10.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.874953 23199 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.005s
10:44:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.879777 23226 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.881075 23199 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.881356 23199 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdd21cb83af541e895b0058bf4e0b6d2"
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:38 on dist-test-slave-gw5c"
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.883216 23199 fs_report.cc:389] FS layout report
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/data
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:44:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:44:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:44:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:44:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:44:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:44:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:44:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:44:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:44:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.961450 23199 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:44:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.963557 23199 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.964138 23199 kserver.cc:163] Server-wide thread pool size limit: 3276
10:44:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.967618 23199 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:44:10.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.974267 23241 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
10:44:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.983083 23199 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
10:44:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.983312 23199 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
10:44:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.983605 23199 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
10:44:10.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.988279 23199 ts_tablet_manager.cc:616] Registered 1 tablets
10:44:10.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.988456 23199 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
10:44:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:10.989109 23241 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Bootstrap starting.
10:44:11.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.053158 23241 log.cc:826] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Log is configured to *not* fsync() on all Append() calls
10:44:11.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.127379 23241 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:44:11.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.128362 23241 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Bootstrap complete.
10:44:11.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.129828 23241 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Time spent bootstrapping tablet: real 0.141s	user 0.085s	sys 0.050s
10:44:11.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.146728 23241 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:11.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.149526 23241 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdd21cb83af541e895b0058bf4e0b6d2, State: Initialized, Role: FOLLOWER
10:44:11.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.150940 23241 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.154316 23241 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Time spent starting tablet: real 0.024s	user 0.023s	sys 0.003s
10:44:11.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.186335 23199 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.193:34951
10:44:11.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.186714 23412 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.193:34951 every 8 connection(s)
10:44:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.188961 23199 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:44:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.196376 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 23199
10:44:11.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:11.232 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:44:11.233 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.36.194:39729
10:44:11.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

10:44:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.234704 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 22610
10:44:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.244385 23413 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:44:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.244951 23413 heartbeater.cc:461] Registering TS with master...
10:44:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.246570 23415 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:44:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.247277 23415 heartbeater.cc:461] Registering TS with master...
10:44:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.249624 23415 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:44:11.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.246598 23413 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:44:11.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.257675 20698 ts_manager.cc:194] Re-registered known tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:11.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.259238 20951 ts_manager.cc:194] Re-registered known tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:11.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.261104 23414 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:44:11.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.261169 20698 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.193:52735
10:44:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.261483 23414 heartbeater.cc:461] Registering TS with master...
10:44:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.262382 23414 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:44:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:11.266278 20825 ts_manager.cc:194] Re-registered known tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:11.294 [DEBUG - kudu-nio-0] (Connection.java:239) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] handling channelInactive
10:44:11.294 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state READY due to: connection closed
10:44:11.295 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:11.295 [DEBUG - Time-limited test] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:11.297 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.297 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.297 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.304 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:11.304 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:11.305 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [9ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@913080894(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@75184c0e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c0e46dd -> wakeup thread Time-limited test)) after lookup
10:44:11.305 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:11.306 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.306 [DEBUG - kudu-nio-2] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.306 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.324 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:11.324 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:11.325 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [9ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [10ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [11ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [11ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@913080894(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@75184c0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c0e46dd -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449))) after lookup
10:44:11.325 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:11.326 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.326 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.327 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.364 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:11.365 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:11.365 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=70), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [9ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [10ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [11ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [11ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [30ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [31ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [69ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@913080894(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@75184c0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449) -> (continuation of Deferred@1354205479 after retry RPC@958438318), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c0e46dd -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449) -> (continuation of Deferred@1354205479 after retry RPC@958438318))) after lookup
10:44:11.366 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:11.367 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.368 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.368 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.404 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:11.405 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:11.405 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=110), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [9ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [10ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [11ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [11ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [30ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [31ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [69ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [71ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [73ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [73ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [109ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@913080894(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@75184c0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449) -> (continuation of Deferred@1354205479 after retry RPC@958438318) -> (continuation of Deferred@1479365235 after retry RPC@1710051123), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c0e46dd -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449) -> (continuation of Deferred@1354205479 after retry RPC@958438318) -> (continuation of Deferred@1479365235 after retry RPC@1710051123))) after lookup
10:44:11.406 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:11.407 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.407 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.407 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.784 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:11.784 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:11.785 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=489), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [9ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [10ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [11ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [11ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [30ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [31ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [69ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [71ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [73ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [73ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [109ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [111ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [112ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [113ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [488ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@913080894(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@75184c0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449) -> (continuation of Deferred@1354205479 after retry RPC@958438318) -> (continuation of Deferred@1479365235 after retry RPC@1710051123) -> (continuation of Deferred@1440713307 after retry RPC@1514459908), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c0e46dd -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449) -> (continuation of Deferred@1354205479 after retry RPC@958438318) -> (continuation of Deferred@1479365235 after retry RPC@1710051123) -> (continuation of Deferred@1440713307 after retry RPC@1514459908))) after lookup
10:44:11.785 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:11.786 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.786 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.786 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.844 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:11.845 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:11.845 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=550), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [9ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [10ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [11ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [11ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [30ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [31ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [69ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [71ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [73ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [73ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [109ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [111ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [112ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [113ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [488ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [490ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [491ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [491ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [545ms] refreshing cache from master, [545ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [549ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@913080894(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@75184c0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449) -> (continuation of Deferred@1354205479 after retry RPC@958438318) -> (continuation of Deferred@1479365235 after retry RPC@1710051123) -> (continuation of Deferred@1440713307 after retry RPC@1514459908) -> (continuation of Deferred@2023177758 after retry RPC@79109883), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c0e46dd -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449) -> (continuation of Deferred@1354205479 after retry RPC@958438318) -> (continuation of Deferred@1479365235 after retry RPC@1710051123) -> (continuation of Deferred@1440713307 after retry RPC@1514459908) -> (continuation of Deferred@2023177758 after retry RPC@79109883))) after lookup
10:44:11.846 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:11.847 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.847 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:11.848 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:12.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.265347 23415 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:44:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.440008 23435 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 6 FOLLOWER]: Starting pre-election (detected failure of leader c5373a0a37e14c38bc09e40a3616b123)
10:44:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.440430 23435 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.442695 23435 leader_election.cc:290] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:12.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:12.445966 22946 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111)
10:44:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:12.451205 22946 leader_election.cc:336] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111)
10:44:12.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.455906 23334 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "a766ec61926c4e27b86cde01c49ea7ed" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" is_pre_election: true
10:44:12.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.456660 23334 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a766ec61926c4e27b86cde01c49ea7ed in term 5.
10:44:12.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.458082 22946 leader_election.cc:304] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, cdd21cb83af541e895b0058bf4e0b6d2; no voters: c5373a0a37e14c38bc09e40a3616b123
10:44:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.458712 23435 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 6 FOLLOWER]: Leader pre-election won for term 7
10:44:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.459019 23435 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 6 FOLLOWER]: Starting leader election (detected failure of leader c5373a0a37e14c38bc09e40a3616b123)
10:44:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.459363 23435 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 6 FOLLOWER]: Advancing to term 7
10:44:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.464143 23435 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.465656 23435 leader_election.cc:290] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 7 election: Requested vote from peers c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:12.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.467351 23334 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "a766ec61926c4e27b86cde01c49ea7ed" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "cdd21cb83af541e895b0058bf4e0b6d2"
10:44:12.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.467935 23334 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 5 FOLLOWER]: Advancing to term 7
10:44:12.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:12.469236 22946 leader_election.cc:336] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111)
10:44:12.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.474465 23334 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a766ec61926c4e27b86cde01c49ea7ed in term 7.
10:44:12.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.475606 22946 leader_election.cc:304] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, cdd21cb83af541e895b0058bf4e0b6d2; no voters: c5373a0a37e14c38bc09e40a3616b123
10:44:12.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.476226 23435 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 7 FOLLOWER]: Leader election won for term 7
10:44:12.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.477728 23435 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 7 LEADER]: Becoming Leader. State: Replica: a766ec61926c4e27b86cde01c49ea7ed, State: Running, Role: LEADER
10:44:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.478554 23435 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:12.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.486804 20698 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed reported cstate change: term changed from 6 to 7, leader changed from c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194) to a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195). New cstate: current_term: 7 leader_uuid: "a766ec61926c4e27b86cde01c49ea7ed" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: UNKNOWN } } }
10:44:12.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.906033 23334 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 7 FOLLOWER]: Refusing update from remote peer a766ec61926c4e27b86cde01c49ea7ed: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
10:44:12.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:12.907763 23435 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [LEADER]: Connected to new peer: Peer: permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
10:44:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:12.954459 22946 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed -> Peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Couldn't send request to peer c5373a0a37e14c38bc09e40a3616b123. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:44:14.144 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:14.144 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:14.145 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2849), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [9ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [10ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [11ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [11ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [30ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [31ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [69ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [71ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [73ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [73ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [109ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [111ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [112ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [113ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [488ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [490ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [491ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [491ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [545ms] refreshing cache from master, [545ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [549ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [551ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [552ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [553ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2845ms] refreshing cache from master, [2845ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [2849ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@913080894(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@75184c0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449) -> (continuation of Deferred@1354205479 after retry RPC@958438318) -> (continuation of Deferred@1479365235 after retry RPC@1710051123) -> (continuation of Deferred@1440713307 after retry RPC@1514459908) -> (continuation of Deferred@2023177758 after retry RPC@79109883) -> (continuation of Deferred@1655250270 after retry RPC@1591052086), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c0e46dd -> wakeup thread Time-limited test -> (continuation of Deferred@332739405 after retry RPC@545865449) -> (continuation of Deferred@1354205479 after retry RPC@958438318) -> (continuation of Deferred@1479365235 after retry RPC@1710051123) -> (continuation of Deferred@1440713307 after retry RPC@1514459908) -> (continuation of Deferred@2023177758 after retry RPC@79109883) -> (continuation of Deferred@1655250270 after retry RPC@1591052086))) after lookup
10:44:14.145 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:14.146 [DEBUG - kudu-nio-0] (Connection.java:757) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] Successfully connected to peer
10:44:14.168 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x1e9bd979, L:/127.0.0.1:55194 - R:/127.20.36.195:44799] using TOKEN/null
10:44:14.225 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.36.194:39729
10:44:14.225 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

10:44:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:14.227315 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data
10:44:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/logs
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.194:39729
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.194
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45775
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.194
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:44:14.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:14.543751 23481 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:44:14.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:14.544229 23481 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:44:14.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:14.544685 23481 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:44:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:14.579633 23481 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:44:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:14.580660 23481 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.194
10:44:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:14.617408 23481 tablet_server_runner.cc:78] Tablet server non-default flags:
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.194:39729
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.194
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45775
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.194
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/logs
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:44:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:44:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:44:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:14.618717 23481 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:44:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:14.620652 23481 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:44:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:14.633782 23494 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:15.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:15.548400 22946 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed -> Peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Couldn't send request to peer c5373a0a37e14c38bc09e40a3616b123. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:44:16.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:16.039225 23493 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 23481
10:44:16.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:16.043123 23493 kernel_stack_watchdog.cc:198] Thread 23481 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 402ms:
10:44:16.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
10:44:16.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
10:44:16.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
10:44:16.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
10:44:16.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:16.043315 23481 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.409s	user 0.504s	sys 0.887s
10:44:16.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:16.043747 23481 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.410s	user 0.504s	sys 0.887s
10:44:16.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:14.634550 23495 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:16.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:16.044610 23496 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1408 milliseconds
10:44:16.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:16.045830 23498 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:16.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.045892 23481 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:44:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.049150 23481 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:44:16.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.051874 23481 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:44:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.053294 23481 hybrid_clock.cc:648] HybridClock initialized: now 1786445056053256 us; error 36 us; skew 500 ppm
10:44:16.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.054193 23481 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:44:16.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.060545 23481 webserver.cc:533] Webserver started at http://127.20.36.194:45775/ using document root <none> and password file <none>
10:44:16.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.061712 23481 fs_manager.cc:362] Metadata directory not provided
10:44:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.062009 23481 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:44:16.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.069909 23481 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
10:44:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.074543 23517 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:44:16.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.075657 23481 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
10:44:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.076016 23481 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:44:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:40 on dist-test-slave-gw5c"
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.077859 23481 fs_report.cc:389] FS layout report
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/data
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:44:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:44:16.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:44:16.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:44:16.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:44:16.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:44:16.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:44:16.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.140477 23481 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:44:16.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.142110 23481 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:44:16.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.142614 23481 kserver.cc:163] Server-wide thread pool size limit: 3276
10:44:16.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.145390 23481 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:44:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.151798 23530 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
10:44:16.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.160073 23481 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
10:44:16.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.160287 23481 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.003s
10:44:16.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.160547 23481 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
10:44:16.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.165333 23481 ts_tablet_manager.cc:616] Registered 1 tablets
10:44:16.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.165586 23481 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
10:44:16.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.165982 23530 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Bootstrap starting.
10:44:16.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.245867 23530 log.cc:826] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Log is configured to *not* fsync() on all Append() calls
10:44:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.344812 23530 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:44:16.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.345750 23530 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Bootstrap complete.
10:44:16.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.347242 23530 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Time spent bootstrapping tablet: real 0.182s	user 0.133s	sys 0.037s
10:44:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.362706 23530 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.366190 23530 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5373a0a37e14c38bc09e40a3616b123, State: Initialized, Role: FOLLOWER
10:44:16.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.367436 23530 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:16.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.372577 23530 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Time spent starting tablet: real 0.025s	user 0.023s	sys 0.001s
10:44:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.382513 23481 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.194:39729
10:44:16.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.382802 23683 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.194:39729 every 8 connection(s)
10:44:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.385397 23481 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:44:16.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.391817 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 23481
10:44:16.394 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:16.448 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:44:16.450 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.36.195:44799
10:44:16.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

10:44:16.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.448973 23690 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:44:16.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.449536 23690 heartbeater.cc:461] Registering TS with master...
10:44:16.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.450934 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 22910
10:44:16.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.451007 23690 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:44:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.453775 23689 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:44:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.454130 23689 heartbeater.cc:461] Registering TS with master...
10:44:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.454973 23689 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:44:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.458014 20698 ts_manager.cc:194] Re-registered known tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.458926 20824 ts_manager.cc:194] Re-registered known tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:16.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.460003 23688 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:44:16.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.460367 23688 heartbeater.cc:461] Registering TS with master...
10:44:16.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.460628 20698 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.194:55871
10:44:16.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.461349 23688 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:44:16.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:16.464998 20951 ts_manager.cc:194] Re-registered known tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:16.487 [DEBUG - kudu-nio-0] (Connection.java:239) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] handling channelInactive
10:44:16.488 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state READY due to: connection closed
10:44:16.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:16.489 [DEBUG - Time-limited test] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:16.490 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.490 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.503 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:16.504 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:16.504 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [2ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@449910350(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@684fde15 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@712046ef -> wakeup thread Time-limited test)) after lookup
10:44:16.504 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:16.505 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.505 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.505 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.524 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:16.524 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:16.524 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [2ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [17ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [17ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@449910350(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@684fde15 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@547738025 after retry RPC@1767904533), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@712046ef -> wakeup thread Time-limited test -> (continuation of Deferred@547738025 after retry RPC@1767904533))) after lookup
10:44:16.525 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:16.526 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.526 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.526 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.544 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:16.544 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:16.544 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [2ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [17ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [17ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [37ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [38ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [38ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [55ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@449910350(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@684fde15 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@547738025 after retry RPC@1767904533) -> (continuation of Deferred@893282899 after retry RPC@1545537884), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@712046ef -> wakeup thread Time-limited test -> (continuation of Deferred@547738025 after retry RPC@1767904533) -> (continuation of Deferred@893282899 after retry RPC@1545537884))) after lookup
10:44:16.545 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:16.546 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.546 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.546 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.664 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:16.665 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:16.665 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=176), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [2ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [17ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [17ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [37ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [38ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [38ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [55ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [57ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [58ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [58ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [172ms] refreshing cache from master, [172ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [176ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@449910350(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@684fde15 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@547738025 after retry RPC@1767904533) -> (continuation of Deferred@893282899 after retry RPC@1545537884) -> (continuation of Deferred@1463100721 after retry RPC@1942258530), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@712046ef -> wakeup thread Time-limited test -> (continuation of Deferred@547738025 after retry RPC@1767904533) -> (continuation of Deferred@893282899 after retry RPC@1545537884) -> (continuation of Deferred@1463100721 after retry RPC@1942258530))) after lookup
10:44:16.666 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:16.667 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.667 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:16.667 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:17.124 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:17.124 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:17.124 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=636), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [2ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [17ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [17ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [37ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [38ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [38ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [55ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [57ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [58ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [58ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [172ms] refreshing cache from master, [172ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [176ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [177ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [179ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [179ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [632ms] refreshing cache from master, [632ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [636ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@449910350(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@684fde15 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@547738025 after retry RPC@1767904533) -> (continuation of Deferred@893282899 after retry RPC@1545537884) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@1505307407 after retry RPC@238124233), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@712046ef -> wakeup thread Time-limited test -> (continuation of Deferred@547738025 after retry RPC@1767904533) -> (continuation of Deferred@893282899 after retry RPC@1545537884) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@1505307407 after retry RPC@238124233))) after lookup
10:44:17.125 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:17.126 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:17.126 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:17.126 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:17.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.464768 23690 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:44:17.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.670464 23701 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader a766ec61926c4e27b86cde01c49ea7ed)
10:44:17.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.670948 23701 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:17.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.673388 23701 leader_election.cc:290] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:17.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:17.677259 23229 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.678167 23708 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 6 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:44:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.679121 23708 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:17.681516 23229 leader_election.cc:336] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.682050 23708 leader_election.cc:290] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:17.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:17.687348 23521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:17.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.691175 23620 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "c5373a0a37e14c38bc09e40a3616b123" is_pre_election: true
10:44:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.692126 23620 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cdd21cb83af541e895b0058bf4e0b6d2 in term 6.
10:44:17.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:17.692930 23521 leader_election.cc:336] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:17.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.693966 23229 leader_election.cc:304] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c5373a0a37e14c38bc09e40a3616b123, cdd21cb83af541e895b0058bf4e0b6d2; no voters: a766ec61926c4e27b86cde01c49ea7ed
10:44:17.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.694833 23701 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 7 FOLLOWER]: Leader pre-election won for term 8
10:44:17.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.695168 23701 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 7 FOLLOWER]: Starting leader election (detected failure of leader a766ec61926c4e27b86cde01c49ea7ed)
10:44:17.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.695570 23701 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 7 FOLLOWER]: Advancing to term 8
10:44:17.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.696030 23334 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "c5373a0a37e14c38bc09e40a3616b123" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" is_pre_election: true
10:44:17.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.702003 23701 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:17.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.702919 23334 raft_consensus.cc:2368] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 8 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c5373a0a37e14c38bc09e40a3616b123 for earlier term 7. Current term is 8.
10:44:17.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.703783 23701 leader_election.cc:290] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 8 election: Requested vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.706749 23521 leader_election.cc:400] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 7 pre-election: Vote denied by peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951) with higher term. Message: Invalid argument: T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 8 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c5373a0a37e14c38bc09e40a3616b123 for earlier term 7. Current term is 8.
10:44:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.706629 23620 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:44:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.707005 23521 leader_election.cc:403] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 7 pre-election: Cancelling election due to peer responding with higher term
10:44:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.707185 23620 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 6 FOLLOWER]: Advancing to term 8
10:44:17.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:17.709558 23229 leader_election.cc:336] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:17.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.714354 23620 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cdd21cb83af541e895b0058bf4e0b6d2 in term 8.
10:44:17.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.714857 23708 raft_consensus.cc:2749] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 8 FOLLOWER]: Leader pre-election lost for term 7. Reason: Vote denied by peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951) with higher term. Message: Invalid argument: T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 8 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c5373a0a37e14c38bc09e40a3616b123 for earlier term 7. Current term is 8.
10:44:17.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.715705 23229 leader_election.cc:304] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c5373a0a37e14c38bc09e40a3616b123, cdd21cb83af541e895b0058bf4e0b6d2; no voters: a766ec61926c4e27b86cde01c49ea7ed
10:44:17.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.716442 23701 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 8 FOLLOWER]: Leader election won for term 8
10:44:17.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.719662 23701 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 8 LEADER]: Becoming Leader. State: Replica: cdd21cb83af541e895b0058bf4e0b6d2, State: Running, Role: LEADER
10:44:17.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.720660 23701 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.731649 20698 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 reported cstate change: term changed from 7 to 8, leader changed from a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195) to cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193). New cstate: current_term: 8 leader_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: HEALTHY } } }
10:44:17.964 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:17.964 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:17.965 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1476), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [2ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [17ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [17ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [37ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [38ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [38ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [55ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [57ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [58ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [58ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [172ms] refreshing cache from master, [172ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [176ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [177ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [179ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [179ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [632ms] refreshing cache from master, [632ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [636ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [637ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [638ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [638ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1472ms] refreshing cache from master, [1472ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [1475ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@449910350(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@684fde15 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@547738025 after retry RPC@1767904533) -> (continuation of Deferred@893282899 after retry RPC@1545537884) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@1505307407 after retry RPC@238124233) -> (continuation of Deferred@28750685 after retry RPC@73683094), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@712046ef -> wakeup thread Time-limited test -> (continuation of Deferred@547738025 after retry RPC@1767904533) -> (continuation of Deferred@893282899 after retry RPC@1545537884) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@1505307407 after retry RPC@238124233) -> (continuation of Deferred@28750685 after retry RPC@73683094))) after lookup
10:44:17.965 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:17.966 [DEBUG - kudu-nio-7] (Connection.java:757) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] Successfully connected to peer
10:44:17.984 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0x10f9b4fc, L:/127.0.0.1:58404 - R:/127.20.36.193:34951] using TOKEN/null
10:44:17.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.994112 23620 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 8 FOLLOWER]: Refusing update from remote peer cdd21cb83af541e895b0058bf4e0b6d2: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 30. (index mismatch)
10:44:17.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:17.996191 23720 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.001s
10:44:17.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:17.996811 23229 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 -> Peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Couldn't send request to peer a766ec61926c4e27b86cde01c49ea7ed. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:44:18.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:18.018602 23736 mvcc.cc:204] Tried to move back new op lower bound from 7317278957527592960 to 7317278956436467712. Current Snapshot: MvccSnapshot[applied={T|T < 7317278957527592960}]
10:44:18.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:18.028784 23739 mvcc.cc:204] Tried to move back new op lower bound from 7317278957527592960 to 7317278956436467712. Current Snapshot: MvccSnapshot[applied={T|T < 7317278957527592960 or (T in {7317278957527592960})}]
10:44:18.058 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.36.195:44799
10:44:18.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:18.060766 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/logs
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.195:44799
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.195
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44795
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.195
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:18.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:44:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:18.383307 23747 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:44:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:18.383881 23747 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:44:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:18.384599 23747 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:44:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:18.418908 23747 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:44:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:18.419889 23747 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.195
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:18.457119 23747 tablet_server_runner.cc:78] Tablet server non-default flags:
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.195:44799
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.195
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44795
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.195
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/logs
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:44:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:18.458415 23747 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:44:18.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:18.460044 23747 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:44:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:18.472426 23755 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:19.555197 20873 debug-util.cc:398] Leaking SignalData structure 0x7b08000a7dc0 after lost signal to thread 20772
10:44:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:19.556222 20873 debug-util.cc:398] Leaking SignalData structure 0x7b080003fa20 after lost signal to thread 20879
10:44:19.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:18.473277 23756 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:19.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:19.877527 23754 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 23747
10:44:19.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:19.941596 23747 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.469s	user 0.434s	sys 1.011s
10:44:19.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:19.944168 23758 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:19.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:19.944396 23747 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.472s	user 0.434s	sys 1.012s
10:44:19.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:19.947659 23757 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1469 milliseconds
10:44:19.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.947718 23747 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:44:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.949642 23747 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:44:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.951968 23747 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:44:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.953300 23747 hybrid_clock.cc:648] HybridClock initialized: now 1786445059953270 us; error 33 us; skew 500 ppm
10:44:19.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.954077 23747 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:44:19.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.960359 23747 webserver.cc:533] Webserver started at http://127.20.36.195:44795/ using document root <none> and password file <none>
10:44:19.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.961252 23747 fs_manager.cc:362] Metadata directory not provided
10:44:19.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.961441 23747 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:44:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.969493 23747 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.004s	sys 0.000s
10:44:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.974172 23777 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:44:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.975382 23747 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.000s
10:44:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.975772 23747 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a766ec61926c4e27b86cde01c49ea7ed"
10:44:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:42 on dist-test-slave-gw5c"
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:19.977674 23747 fs_report.cc:389] FS layout report
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/data
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:44:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:44:20.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.034216 23747 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:44:20.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.035941 23747 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:44:20.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.036540 23747 kserver.cc:163] Server-wide thread pool size limit: 3276
10:44:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.041898 23747 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:44:20.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.050244 23793 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
10:44:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.062858 23747 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
10:44:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.063202 23747 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.015s	user 0.001s	sys 0.001s
10:44:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.063575 23747 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
10:44:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.070803 23747 ts_tablet_manager.cc:616] Registered 1 tablets
10:44:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.071358 23747 ts_tablet_manager.cc:595] Time spent register tablets: real 0.008s	user 0.007s	sys 0.000s
10:44:20.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.071568 23793 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap starting.
10:44:20.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.153798 23793 log.cc:826] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Log is configured to *not* fsync() on all Append() calls
10:44:20.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.255458 23793 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:44:20.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.256486 23793 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap complete.
10:44:20.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.258345 23793 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Time spent bootstrapping tablet: real 0.187s	user 0.146s	sys 0.036s
10:44:20.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.280546 23793 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:20.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.284444 23793 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: a766ec61926c4e27b86cde01c49ea7ed, State: Initialized, Role: FOLLOWER
10:44:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.285683 23793 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:20.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.292517 23793 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Time spent starting tablet: real 0.034s	user 0.028s	sys 0.004s
10:44:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.361745 23747 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.195:44799
10:44:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.362116 23962 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.195:44799 every 8 connection(s)
10:44:20.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.365829 23747 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:44:20.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.375638 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 23747
10:44:20.376 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:20.425 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:44:20.427 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.36.193:34951
10:44:20.427 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

10:44:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.428292 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 23199
10:44:20.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.433853 23965 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:44:20.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.434556 23965 heartbeater.cc:461] Registering TS with master...
10:44:20.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.436338 23965 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:44:20.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.442101 20698 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:44:20.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.446214 20698 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.195:47773
10:44:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.468713 23963 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:44:20.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.469132 23963 heartbeater.cc:461] Registering TS with master...
10:44:20.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.469913 23963 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:44:20.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.473507 20951 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:44:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.483570 23964 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:44:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.483997 23964 heartbeater.cc:461] Registering TS with master...
10:44:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.484730 23964 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:44:20.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:20.489430 20825 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:44:20.495 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:20.497 [DEBUG - kudu-nio-7] (Connection.java:239) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] handling channelInactive
10:44:20.497 [DEBUG - kudu-nio-7] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state READY due to: connection closed
10:44:20.498 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: connection closed
10:44:20.504 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:20.505 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:20.505 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [8ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1351640333(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@76566f69 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@430e36de -> wakeup thread Time-limited test)) after lookup
10:44:20.505 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:20.507 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.507 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.507 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.524 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:20.524 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:20.524 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [8ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [9ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [11ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [11ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1351640333(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@76566f69 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1901443400 after retry RPC@1138743674), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@430e36de -> wakeup thread Time-limited test -> (continuation of Deferred@1901443400 after retry RPC@1138743674))) after lookup
10:44:20.525 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:20.526 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.526 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.565 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:20.565 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:20.565 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [8ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [9ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [11ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [11ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [28ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [30ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [30ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [68ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1351640333(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@76566f69 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1901443400 after retry RPC@1138743674) -> (continuation of Deferred@297197834 after retry RPC@748682831), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@430e36de -> wakeup thread Time-limited test -> (continuation of Deferred@1901443400 after retry RPC@1138743674) -> (continuation of Deferred@297197834 after retry RPC@748682831))) after lookup
10:44:20.566 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:20.567 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.567 [DEBUG - kudu-nio-2] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.567 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.624 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:20.624 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:20.625 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=128), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [8ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [9ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [11ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [11ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [28ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [30ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [30ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [68ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [69ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [71ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [71ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [128ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1351640333(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@76566f69 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1901443400 after retry RPC@1138743674) -> (continuation of Deferred@297197834 after retry RPC@748682831) -> (continuation of Deferred@1157542557 after retry RPC@32509766), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@430e36de -> wakeup thread Time-limited test -> (continuation of Deferred@1901443400 after retry RPC@1138743674) -> (continuation of Deferred@297197834 after retry RPC@748682831) -> (continuation of Deferred@1157542557 after retry RPC@32509766))) after lookup
10:44:20.625 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:20.626 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.626 [DEBUG - kudu-nio-3] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.626 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.824 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:20.824 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:20.825 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=329), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [8ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [9ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [11ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [11ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [28ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [30ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [30ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [68ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [69ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [71ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [71ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [128ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [129ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [130ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [130ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [324ms] refreshing cache from master, [324ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [328ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1351640333(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@76566f69 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1901443400 after retry RPC@1138743674) -> (continuation of Deferred@297197834 after retry RPC@748682831) -> (continuation of Deferred@1157542557 after retry RPC@32509766) -> (continuation of Deferred@352144478 after retry RPC@368503736), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@430e36de -> wakeup thread Time-limited test -> (continuation of Deferred@1901443400 after retry RPC@1138743674) -> (continuation of Deferred@297197834 after retry RPC@748682831) -> (continuation of Deferred@1157542557 after retry RPC@32509766) -> (continuation of Deferred@352144478 after retry RPC@368503736))) after lookup
10:44:20.826 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:20.827 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.827 [DEBUG - kudu-nio-4] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:20.827 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951) for tablet d339d350474b44a98222592091597860: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951
10:44:21.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.451049 23965 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:44:21.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.717139 23981 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader cdd21cb83af541e895b0058bf4e0b6d2)
10:44:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.717600 23981 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.719252 23981 leader_election.cc:290] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:21.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:21.726452 23521 leader_election.cc:336] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111)
10:44:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.731067 23895 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "c5373a0a37e14c38bc09e40a3616b123" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "a766ec61926c4e27b86cde01c49ea7ed" is_pre_election: true
10:44:21.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.731868 23895 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c5373a0a37e14c38bc09e40a3616b123 in term 7.
10:44:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.733105 23521 leader_election.cc:304] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, c5373a0a37e14c38bc09e40a3616b123; no voters: cdd21cb83af541e895b0058bf4e0b6d2
10:44:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.733825 23981 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 8 FOLLOWER]: Leader pre-election won for term 9
10:44:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.734157 23981 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 8 FOLLOWER]: Starting leader election (detected failure of leader cdd21cb83af541e895b0058bf4e0b6d2)
10:44:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.734457 23981 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 8 FOLLOWER]: Advancing to term 9
10:44:21.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.738756 23981 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:21.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.740370 23981 leader_election.cc:290] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 9 election: Requested vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:21.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.741071 23895 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "c5373a0a37e14c38bc09e40a3616b123" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "a766ec61926c4e27b86cde01c49ea7ed"
10:44:21.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.741462 23895 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 7 FOLLOWER]: Advancing to term 9
10:44:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:21.744592 23521 leader_election.cc:336] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111)
10:44:21.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.748045 23895 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c5373a0a37e14c38bc09e40a3616b123 in term 9.
10:44:21.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.749049 23521 leader_election.cc:304] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, c5373a0a37e14c38bc09e40a3616b123; no voters: cdd21cb83af541e895b0058bf4e0b6d2
10:44:21.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.749663 23981 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 9 FOLLOWER]: Leader election won for term 9
10:44:21.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.750838 23981 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 9 LEADER]: Becoming Leader. State: Replica: c5373a0a37e14c38bc09e40a3616b123, State: Running, Role: LEADER
10:44:21.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.751797 23981 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:21.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.761166 20698 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 reported cstate change: term changed from 8 to 9, leader changed from cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193) to c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194). New cstate: current_term: 9 leader_uuid: "c5373a0a37e14c38bc09e40a3616b123" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: UNKNOWN } } }
10:44:21.803 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:21.804 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:21.804 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1308), Traces: [0ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [1ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [8ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [9ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [11ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [11ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [27ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [28ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [30ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [30ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [68ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [69ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [71ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [71ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [128ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [129ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [130ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [130ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [324ms] refreshing cache from master, [324ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [328ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [330ms] sending RPC to server cdd21cb83af541e895b0058bf4e0b6d2, [331ms] received response from server cdd21cb83af541e895b0058bf4e0b6d2: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [331ms] delaying RPC due to: Network error: Failed to connect to peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951): Connection refused: /127.20.36.193:34951, [1304ms] refreshing cache from master, [1304ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [1307ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1351640333(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@76566f69 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1901443400 after retry RPC@1138743674) -> (continuation of Deferred@297197834 after retry RPC@748682831) -> (continuation of Deferred@1157542557 after retry RPC@32509766) -> (continuation of Deferred@352144478 after retry RPC@368503736) -> (continuation of Deferred@324418356 after retry RPC@828357069), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@430e36de -> wakeup thread Time-limited test -> (continuation of Deferred@1901443400 after retry RPC@1138743674) -> (continuation of Deferred@297197834 after retry RPC@748682831) -> (continuation of Deferred@1157542557 after retry RPC@32509766) -> (continuation of Deferred@352144478 after retry RPC@368503736) -> (continuation of Deferred@324418356 after retry RPC@828357069))) after lookup
10:44:21.805 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:21.812 [DEBUG - kudu-nio-5] (Connection.java:757) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] Successfully connected to peer
10:44:21.836 [DEBUG - kudu-nio-5] (Negotiator.java:918) Authenticated connection [id: 0x5a616b01, L:/127.0.0.1:37658 - R:/127.20.36.194:39729] using TOKEN/null
10:44:21.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.845863 23895 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 9 FOLLOWER]: Refusing update from remote peer c5373a0a37e14c38bc09e40a3616b123: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 34. (index mismatch)
10:44:21.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:21.846647 23521 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 -> Peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Couldn't send request to peer cdd21cb83af541e895b0058bf4e0b6d2. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:44:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.847622 23989 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
10:44:21.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.873930 24014 mvcc.cc:204] Tried to move back new op lower bound from 7317278973302546432 to 7317278972942913536. Current Snapshot: MvccSnapshot[applied={T|T < 7317278973302546432}]
10:44:21.910 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.36.193:34951
10:44:21.910 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:21.911670 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/logs
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.193:34951
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.193
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35283
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.193
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:44:22.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:22.237311 24026 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:44:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:22.237802 24026 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:44:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:22.238268 24026 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:44:22.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:22.275163 24026 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:44:22.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:22.276202 24026 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.193
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:22.314872 24026 tablet_server_runner.cc:78] Tablet server non-default flags:
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.193:34951
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.193
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35283
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.193
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/logs
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:44:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:44:22.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:22.316780 24026 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:44:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:22.318604 24026 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:44:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:22.332860 24038 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:22.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:22.869387 23521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111) [suppressed 4 similar messages]
10:44:24.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:23.736569 24034 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 24026
10:44:24.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:22.333312 24039 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:24.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:24.205477 24034 kernel_stack_watchdog.cc:198] Thread 24026 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 400ms:
10:44:24.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
10:44:24.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
10:44:24.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
10:44:24.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
10:44:24.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:24.206178 24026 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.874s	user 0.620s	sys 1.157s
10:44:24.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:24.206431 24026 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.874s	user 0.620s	sys 1.157s
10:44:24.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:24.207768 24041 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:24.210579 24040 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1871 milliseconds
10:44:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.210608 24026 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:44:24.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.211972 24026 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:44:24.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.215664 24026 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:44:24.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.217096 24026 hybrid_clock.cc:648] HybridClock initialized: now 1786445064217082 us; error 64 us; skew 500 ppm
10:44:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.218161 24026 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:44:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.224647 24026 webserver.cc:533] Webserver started at http://127.20.36.193:35283/ using document root <none> and password file <none>
10:44:24.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.225677 24026 fs_manager.cc:362] Metadata directory not provided
10:44:24.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.225955 24026 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:44:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.234268 24026 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
10:44:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.239085 24057 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:44:24.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.240231 24026 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
10:44:24.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.240619 24026 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:44:24.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdd21cb83af541e895b0058bf4e0b6d2"
10:44:24.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:38 on dist-test-slave-gw5c"
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.242686 24026 fs_report.cc:389] FS layout report
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/wal
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/data
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:44:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:44:24.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:44:24.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:44:24.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:44:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:24.261870 23521 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 -> Peer cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951): Couldn't send request to peer cdd21cb83af541e895b0058bf4e0b6d2. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.193:34951: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:44:24.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.308319 24026 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:44:24.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.309902 24026 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:44:24.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.310348 24026 kserver.cc:163] Server-wide thread pool size limit: 3276
10:44:24.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.313212 24026 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:44:24.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.319517 24075 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
10:44:24.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.328074 24026 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
10:44:24.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.328336 24026 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
10:44:24.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.328564 24026 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
10:44:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.333549 24026 ts_tablet_manager.cc:616] Registered 1 tablets
10:44:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.333863 24026 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
10:44:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.334192 24075 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Bootstrap starting.
10:44:24.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.424139 24075 log.cc:826] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Log is configured to *not* fsync() on all Append() calls
10:44:24.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.520722 24075 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:44:24.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.522127 24075 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Bootstrap complete.
10:44:24.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.524456 24075 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Time spent bootstrapping tablet: real 0.191s	user 0.136s	sys 0.047s
10:44:24.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.544401 24244 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.193:34951 every 8 connection(s)
10:44:24.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.544661 24026 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.193:34951
10:44:24.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.548137 24026 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-0/data/info.pb
10:44:24.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.549553 24075 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:24.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.552325 24075 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdd21cb83af541e895b0058bf4e0b6d2, State: Initialized, Role: FOLLOWER
10:44:24.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.556085 24075 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.558779 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 24026
10:44:24.559 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.595957 24075 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2: Time spent starting tablet: real 0.071s	user 0.035s	sys 0.028s
10:44:24.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.617609 24246 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:44:24.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.618077 24246 heartbeater.cc:461] Registering TS with master...
10:44:24.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.619196 24246 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:44:24.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.624013 20951 ts_manager.cc:194] Re-registered known tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:24.626 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:44:24.627 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.36.194:39729
10:44:24.628 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

10:44:24.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.625960 24248 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:44:24.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.626325 24248 heartbeater.cc:461] Registering TS with master...
10:44:24.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.627280 24248 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:44:24.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.628430 24247 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:44:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.628806 24247 heartbeater.cc:461] Registering TS with master...
10:44:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.629606 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 23481
10:44:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.629743 24247 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:44:24.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.637198 20699 ts_manager.cc:194] Re-registered known tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.638388 20825 ts_manager.cc:194] Re-registered known tserver with Master: cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:24.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.640524 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.193:47931
10:44:24.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:24.644342 24248 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:44:24.667 [DEBUG - kudu-nio-5] (Connection.java:239) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] handling channelInactive
10:44:24.668 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state READY due to: connection closed
10:44:24.668 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:24.669 [DEBUG - Time-limited test] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:24.670 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.670 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.670 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.684 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:24.684 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:24.684 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@249087589(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@420597a2 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d2e4a15 -> wakeup thread Time-limited test)) after lookup
10:44:24.685 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:24.686 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.686 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.686 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.705 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:24.705 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:24.705 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [18ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [18ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@249087589(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@420597a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d2e4a15 -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044))) after lookup
10:44:24.706 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:24.707 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.707 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.707 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.724 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:24.724 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:24.725 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [18ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [18ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [37ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [39ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [39ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [56ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@249087589(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@420597a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044) -> (continuation of Deferred@1305174611 after retry RPC@631986735), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d2e4a15 -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044) -> (continuation of Deferred@1305174611 after retry RPC@631986735))) after lookup
10:44:24.725 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:24.726 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.726 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.804 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:24.805 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:24.805 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=136), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [18ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [18ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [37ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [39ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [39ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [56ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [57ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [58ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [58ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [136ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@249087589(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@420597a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044) -> (continuation of Deferred@1305174611 after retry RPC@631986735) -> (continuation of Deferred@2123746471 after retry RPC@26254200), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d2e4a15 -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044) -> (continuation of Deferred@1305174611 after retry RPC@631986735) -> (continuation of Deferred@2123746471 after retry RPC@26254200))) after lookup
10:44:24.805 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:24.806 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.807 [DEBUG - kudu-nio-2] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.807 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.904 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:24.905 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:24.905 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=236), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [18ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [18ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [37ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [39ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [39ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [56ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [57ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [58ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [58ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [136ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [137ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [139ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [139ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [232ms] refreshing cache from master, [233ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [236ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@249087589(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@420597a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044) -> (continuation of Deferred@1305174611 after retry RPC@631986735) -> (continuation of Deferred@2123746471 after retry RPC@26254200) -> (continuation of Deferred@517224782 after retry RPC@1994013663), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d2e4a15 -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044) -> (continuation of Deferred@1305174611 after retry RPC@631986735) -> (continuation of Deferred@2123746471 after retry RPC@26254200) -> (continuation of Deferred@517224782 after retry RPC@1994013663))) after lookup
10:44:24.906 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:24.907 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.907 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:24.907 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:25.124 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:25.124 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)[L],cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:25.125 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=456), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [18ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [18ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [37ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [39ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [39ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [56ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [57ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [58ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [58ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [136ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [137ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [139ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [139ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [232ms] refreshing cache from master, [233ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [236ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [237ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [239ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [239ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [452ms] refreshing cache from master, [452ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [456ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@249087589(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@420597a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044) -> (continuation of Deferred@1305174611 after retry RPC@631986735) -> (continuation of Deferred@2123746471 after retry RPC@26254200) -> (continuation of Deferred@517224782 after retry RPC@1994013663) -> (continuation of Deferred@1017231605 after retry RPC@453142653), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d2e4a15 -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044) -> (continuation of Deferred@1305174611 after retry RPC@631986735) -> (continuation of Deferred@2123746471 after retry RPC@26254200) -> (continuation of Deferred@517224782 after retry RPC@1994013663) -> (continuation of Deferred@1017231605 after retry RPC@453142653))) after lookup
10:44:25.125 [DEBUG - kudu-nio-2] (Connection.java:748) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] connecting to peer
10:44:25.127 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:25.127 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] cleaning up while in state CONNECTING due to: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:25.127 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729) for tablet d339d350474b44a98222592091597860: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729
10:44:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.687060 24263 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 9 FOLLOWER]: Starting pre-election (detected failure of leader c5373a0a37e14c38bc09e40a3616b123)
10:44:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.687604 24263 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.689810 24263 leader_election.cc:290] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:25.693125 23779 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111)
10:44:25.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:25.696556 23779 leader_election.cc:336] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111)
10:44:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.702075 24171 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "a766ec61926c4e27b86cde01c49ea7ed" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" is_pre_election: true
10:44:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.702841 24171 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a766ec61926c4e27b86cde01c49ea7ed in term 8.
10:44:25.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.704115 23779 leader_election.cc:304] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, cdd21cb83af541e895b0058bf4e0b6d2; no voters: c5373a0a37e14c38bc09e40a3616b123
10:44:25.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.704806 24263 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 9 FOLLOWER]: Leader pre-election won for term 10
10:44:25.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.705063 24263 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 9 FOLLOWER]: Starting leader election (detected failure of leader c5373a0a37e14c38bc09e40a3616b123)
10:44:25.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.705274 24263 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 9 FOLLOWER]: Advancing to term 10
10:44:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.709676 24263 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.711143 24263 leader_election.cc:290] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 10 election: Requested vote from peers c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729), cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193:34951)
10:44:25.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.712714 24171 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "a766ec61926c4e27b86cde01c49ea7ed" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "cdd21cb83af541e895b0058bf4e0b6d2"
10:44:25.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.713254 24171 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 8 FOLLOWER]: Advancing to term 10
10:44:25.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:25.714500 23779 leader_election.cc:336] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111)
10:44:25.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.720276 24171 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a766ec61926c4e27b86cde01c49ea7ed in term 10.
10:44:25.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.721271 23779 leader_election.cc:304] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a766ec61926c4e27b86cde01c49ea7ed, cdd21cb83af541e895b0058bf4e0b6d2; no voters: c5373a0a37e14c38bc09e40a3616b123
10:44:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.721843 24263 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 10 FOLLOWER]: Leader election won for term 10
10:44:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.723445 24263 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 10 LEADER]: Becoming Leader. State: Replica: a766ec61926c4e27b86cde01c49ea7ed, State: Running, Role: LEADER
10:44:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.724341 24263 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:25.733301 20699 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed reported cstate change: term changed from 9 to 10, leader changed from c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194) to a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195). New cstate: current_term: 10 leader_uuid: "a766ec61926c4e27b86cde01c49ea7ed" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: UNKNOWN } } }
10:44:26.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:26.148423 23779 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed -> Peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Couldn't send request to peer c5373a0a37e14c38bc09e40a3616b123. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:44:26.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:26.157706 24171 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 10 FOLLOWER]: Refusing update from remote peer a766ec61926c4e27b86cde01c49ea7ed: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
10:44:26.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:26.159294 24263 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [LEADER]: Connected to new peer: Peer: permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
10:44:26.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:26.174285 24290 mvcc.cc:204] Tried to move back new op lower bound from 7317278973302546432 to 7317278972942913536. Current Snapshot: MvccSnapshot[applied={T|T < 7317278973302546432}]
10:44:27.524 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:27.524 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:27.524 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2856), Traces: [0ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [2ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [16ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [18ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [18ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [37ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [39ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [39ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [56ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [57ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [58ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [58ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [136ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [137ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [139ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [139ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [232ms] refreshing cache from master, [233ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [236ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [237ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [239ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [239ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [452ms] refreshing cache from master, [452ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [456ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [457ms] sending RPC to server c5373a0a37e14c38bc09e40a3616b123, [459ms] received response from server c5373a0a37e14c38bc09e40a3616b123: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [459ms] delaying RPC due to: Network error: Failed to connect to peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729): Connection refused: /127.20.36.194:39729, [2852ms] refreshing cache from master, [2852ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [2856ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@249087589(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@420597a2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044) -> (continuation of Deferred@1305174611 after retry RPC@631986735) -> (continuation of Deferred@2123746471 after retry RPC@26254200) -> (continuation of Deferred@517224782 after retry RPC@1994013663) -> (continuation of Deferred@1017231605 after retry RPC@453142653) -> (continuation of Deferred@1447391907 after retry RPC@1577888150), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d2e4a15 -> wakeup thread Time-limited test -> (continuation of Deferred@2031508009 after retry RPC@1455938044) -> (continuation of Deferred@1305174611 after retry RPC@631986735) -> (continuation of Deferred@2123746471 after retry RPC@26254200) -> (continuation of Deferred@517224782 after retry RPC@1994013663) -> (continuation of Deferred@1017231605 after retry RPC@453142653) -> (continuation of Deferred@1447391907 after retry RPC@1577888150))) after lookup
10:44:27.525 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:27.526 [DEBUG - kudu-nio-5] (Connection.java:757) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] Successfully connected to peer
10:44:27.544 [DEBUG - kudu-nio-5] (Negotiator.java:918) Authenticated connection [id: 0xf1fd229f, L:/127.0.0.1:59580 - R:/127.20.36.195:44799] using TOKEN/null
10:44:27.592 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.36.194:39729
10:44:27.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:27.594134 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/logs
10:44:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.194:39729
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.194
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45775
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.194
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:44:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:27.919862 24316 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:44:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:27.920358 24316 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:44:27.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:27.920908 24316 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:44:27.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:27.961594 24316 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:44:27.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:27.962446 24316 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.194
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:27.998441 24316 tablet_server_runner.cc:78] Tablet server non-default flags:
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.194:39729
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.194
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45775
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.194
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/logs
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:44:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:44:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:27.999835 24316 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:44:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:28.001405 24316 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:44:28.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:28.013724 24323 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:28.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:28.693397 23779 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed -> Peer c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729): Couldn't send request to peer c5373a0a37e14c38bc09e40a3616b123. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.194:39729: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:44:29.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:28.015476 24324 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:29.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:28.018561 24326 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:29.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:29.234859 24325 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
10:44:29.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.235026 24316 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:44:29.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.240281 24316 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:44:29.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.242688 24316 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:44:29.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.244189 24316 hybrid_clock.cc:648] HybridClock initialized: now 1786445069244147 us; error 46 us; skew 500 ppm
10:44:29.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.245061 24316 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:44:29.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.251775 24316 webserver.cc:533] Webserver started at http://127.20.36.194:45775/ using document root <none> and password file <none>
10:44:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.252763 24316 fs_manager.cc:362] Metadata directory not provided
10:44:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.253041 24316 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:44:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.261634 24316 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.001s	sys 0.004s
10:44:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.267088 24346 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:44:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.268363 24316 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
10:44:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.268662 24316 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:44:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:40 on dist-test-slave-gw5c"
10:44:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.271009 24316 fs_report.cc:389] FS layout report
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/wal
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/data
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:44:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:44:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:44:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:44:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:44:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.340142 24316 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:44:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.341773 24316 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:44:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.342201 24316 kserver.cc:163] Server-wide thread pool size limit: 3276
10:44:29.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.348116 24316 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:44:29.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.356187 24359 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
10:44:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.366993 24316 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
10:44:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.367237 24316 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.013s	user 0.000s	sys 0.002s
10:44:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.367647 24316 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
10:44:29.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.375090 24316 ts_tablet_manager.cc:616] Registered 1 tablets
10:44:29.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.375651 24316 ts_tablet_manager.cc:595] Time spent register tablets: real 0.008s	user 0.002s	sys 0.004s
10:44:29.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.376293 24359 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Bootstrap starting.
10:44:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.481777 24359 log.cc:826] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Log is configured to *not* fsync() on all Append() calls
10:44:29.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.592730 24359 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:44:29.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.593645 24359 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Bootstrap complete.
10:44:29.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.595280 24359 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Time spent bootstrapping tablet: real 0.220s	user 0.153s	sys 0.054s
10:44:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.603165 24316 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.194:39729
10:44:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.603323 24520 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.194:39729 every 8 connection(s)
10:44:29.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.605846 24316 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-1/data/info.pb
10:44:29.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.609174 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 24316
10:44:29.611 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.616119 24359 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.619685 24359 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5373a0a37e14c38bc09e40a3616b123, State: Initialized, Role: FOLLOWER
10:44:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.620688 24359 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.634347 24359 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123: Time spent starting tablet: real 0.039s	user 0.025s	sys 0.011s
10:44:29.644 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:44:29.649 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.36.195:44799
10:44:29.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

10:44:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.650316 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 23747
10:44:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.679334 24524 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:44:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.679980 24524 heartbeater.cc:461] Registering TS with master...
10:44:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.681213 24521 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:44:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.681558 24521 heartbeater.cc:461] Registering TS with master...
10:44:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.682497 24521 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:44:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.681465 24524 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:44:29.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.685611 24523 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:44:29.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.685950 24523 heartbeater.cc:461] Registering TS with master...
10:44:29.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.686827 24523 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:44:29.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.687054 20698 ts_manager.cc:194] Re-registered known tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:29.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.687182 20951 ts_manager.cc:194] Re-registered known tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:29.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.690068 20698 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.194:36251
10:44:29.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.692344 20824 ts_manager.cc:194] Re-registered known tserver with Master: c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:29.693969 24524 heartbeater.cc:499] Master 127.20.36.254:38609 was elected leader, sending a full tablet report...
10:44:29.701 [DEBUG - kudu-nio-5] (Connection.java:239) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] handling channelInactive
10:44:29.701 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state READY due to: connection closed
10:44:29.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:29.702 [DEBUG - Time-limited test] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:29.703 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.703 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.703 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.724 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:29.724 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:29.724 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1675696629(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@581a4c06 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@57ef4f23 -> wakeup thread Time-limited test)) after lookup
10:44:29.725 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:29.726 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.726 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.726 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.744 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:29.744 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:29.745 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [24ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [25ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1675696629(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@581a4c06 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1298797301 after retry RPC@1615977004), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@57ef4f23 -> wakeup thread Time-limited test -> (continuation of Deferred@1298797301 after retry RPC@1615977004))) after lookup
10:44:29.745 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:29.746 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.746 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.746 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.763 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:29.764 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:29.764 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [24ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [25ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [44ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [45ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [61ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1675696629(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@581a4c06 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1298797301 after retry RPC@1615977004) -> (continuation of Deferred@979755689 after retry RPC@1418448755), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@57ef4f23 -> wakeup thread Time-limited test -> (continuation of Deferred@1298797301 after retry RPC@1615977004) -> (continuation of Deferred@979755689 after retry RPC@1418448755))) after lookup
10:44:29.765 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:29.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.766 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.824 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:29.824 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:29.824 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=122), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [24ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [25ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [44ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [45ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [61ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [62ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [64ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [64ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [118ms] refreshing cache from master, [118ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [121ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1675696629(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@581a4c06 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1298797301 after retry RPC@1615977004) -> (continuation of Deferred@979755689 after retry RPC@1418448755) -> (continuation of Deferred@586908503 after retry RPC@679555116), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@57ef4f23 -> wakeup thread Time-limited test -> (continuation of Deferred@1298797301 after retry RPC@1615977004) -> (continuation of Deferred@979755689 after retry RPC@1418448755) -> (continuation of Deferred@586908503 after retry RPC@679555116))) after lookup
10:44:29.825 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:29.826 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.826 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:29.826 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:30.043 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:30.044 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)[L],c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)]}]
10:44:30.044 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=342), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [24ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [25ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [44ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [45ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [61ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [62ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [64ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [64ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [118ms] refreshing cache from master, [118ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [121ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [123ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [124ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [124ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [338ms] refreshing cache from master, [338ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [341ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1675696629(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@581a4c06 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1298797301 after retry RPC@1615977004) -> (continuation of Deferred@979755689 after retry RPC@1418448755) -> (continuation of Deferred@586908503 after retry RPC@679555116) -> (continuation of Deferred@1614463496 after retry RPC@813253194), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@57ef4f23 -> wakeup thread Time-limited test -> (continuation of Deferred@1298797301 after retry RPC@1615977004) -> (continuation of Deferred@979755689 after retry RPC@1418448755) -> (continuation of Deferred@586908503 after retry RPC@679555116) -> (continuation of Deferred@1614463496 after retry RPC@813253194))) after lookup
10:44:30.045 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] connecting to peer
10:44:30.045 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:30.045 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] cleaning up while in state CONNECTING due to: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:30.046 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799) for tablet d339d350474b44a98222592091597860: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799
10:44:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.607554 24544 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader a766ec61926c4e27b86cde01c49ea7ed)
10:44:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.607972 24544 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:30.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.610653 24544 leader_election.cc:290] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:30.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:30.615553 24064 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:30.620241 24064 leader_election.cc:336] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.626098 24456 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "c5373a0a37e14c38bc09e40a3616b123" is_pre_election: true
10:44:30.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.626912 24456 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cdd21cb83af541e895b0058bf4e0b6d2 in term 9.
10:44:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.628394 24064 leader_election.cc:304] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c5373a0a37e14c38bc09e40a3616b123, cdd21cb83af541e895b0058bf4e0b6d2; no voters: a766ec61926c4e27b86cde01c49ea7ed
10:44:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.629256 24544 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 10 FOLLOWER]: Leader pre-election won for term 11
10:44:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.629599 24544 raft_consensus.cc:493] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 10 FOLLOWER]: Starting leader election (detected failure of leader a766ec61926c4e27b86cde01c49ea7ed)
10:44:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.629948 24544 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 10 FOLLOWER]: Advancing to term 11
10:44:30.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.637310 24544 raft_consensus.cc:515] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:30.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.638734 24544 leader_election.cc:290] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 11 election: Requested vote from peers a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799), c5373a0a37e14c38bc09e40a3616b123 (127.20.36.194:39729)
10:44:30.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.640697 24456 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "d339d350474b44a98222592091597860" candidate_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "c5373a0a37e14c38bc09e40a3616b123"
10:44:30.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.641299 24456 raft_consensus.cc:3060] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 9 FOLLOWER]: Advancing to term 11
10:44:30.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:30.642800 24064 leader_election.cc:336] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111)
10:44:30.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.648119 24456 raft_consensus.cc:2468] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cdd21cb83af541e895b0058bf4e0b6d2 in term 11.
10:44:30.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.649081 24064 leader_election.cc:304] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c5373a0a37e14c38bc09e40a3616b123, cdd21cb83af541e895b0058bf4e0b6d2; no voters: a766ec61926c4e27b86cde01c49ea7ed
10:44:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.649695 24544 raft_consensus.cc:2804] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 11 FOLLOWER]: Leader election won for term 11
10:44:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.651348 24544 raft_consensus.cc:697] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [term 11 LEADER]: Becoming Leader. State: Replica: cdd21cb83af541e895b0058bf4e0b6d2, State: Running, Role: LEADER
10:44:30.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.652740 24544 consensus_queue.cc:237] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:30.662920 20698 catalog_manager.cc:5719] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 reported cstate change: term changed from 10 to 11, leader changed from a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195) to cdd21cb83af541e895b0058bf4e0b6d2 (127.20.36.193). New cstate: current_term: 11 leader_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } health_report { overall_health: HEALTHY } } }
10:44:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:31.174011 24064 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 -> Peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Couldn't send request to peer a766ec61926c4e27b86cde01c49ea7ed. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:44:31.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:31.253736 24456 raft_consensus.cc:1275] T d339d350474b44a98222592091597860 P c5373a0a37e14c38bc09e40a3616b123 [term 11 FOLLOWER]: Refusing update from remote peer cdd21cb83af541e895b0058bf4e0b6d2: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
10:44:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:31.255163 24568 consensus_queue.cc:1048] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
10:44:31.785 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet d339d350474b44a98222592091597860 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379' with partition [<start>, <end>)
10:44:31.785 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=d339d350474b44a98222592091597860@[a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799),c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729),cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)[L]]}]
10:44:31.785 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=d339d350474b44a98222592091597860, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2083), Traces: [0ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [1ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [1ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [23ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [24ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [25ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [43ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [44ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [45ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [61ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [62ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [64ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [64ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [118ms] refreshing cache from master, [118ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [121ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [123ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [124ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [124ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [338ms] refreshing cache from master, [338ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [341ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, [342ms] sending RPC to server a766ec61926c4e27b86cde01c49ea7ed, [344ms] received response from server a766ec61926c4e27b86cde01c49ea7ed: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [344ms] delaying RPC due to: Network error: Failed to connect to peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799): Connection refused: /127.20.36.195:44799, [2078ms] refreshing cache from master, [2078ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.36.254:38609, [2082ms] Sub RPC GetTableLocations: received response from server master-127.20.36.254:38609: OK, deferred=Deferred@1675696629(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007f973c36b330@581a4c06 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1298797301 after retry RPC@1615977004) -> (continuation of Deferred@979755689 after retry RPC@1418448755) -> (continuation of Deferred@586908503 after retry RPC@679555116) -> (continuation of Deferred@1614463496 after retry RPC@813253194) -> (continuation of Deferred@1368146747 after retry RPC@1756772382), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@57ef4f23 -> wakeup thread Time-limited test -> (continuation of Deferred@1298797301 after retry RPC@1615977004) -> (continuation of Deferred@979755689 after retry RPC@1418448755) -> (continuation of Deferred@586908503 after retry RPC@679555116) -> (continuation of Deferred@1614463496 after retry RPC@813253194) -> (continuation of Deferred@1368146747 after retry RPC@1756772382))) after lookup
10:44:31.786 [DEBUG - kudu-nio-2] (Connection.java:748) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] connecting to peer
10:44:31.787 [DEBUG - kudu-nio-4] (Connection.java:757) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] Successfully connected to peer
10:44:31.815 [DEBUG - kudu-nio-4] (Negotiator.java:918) Authenticated connection [id: 0x6864b954, L:/127.0.0.1:58320 - R:/127.20.36.193:34951] using TOKEN/null
10:44:31.859 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.36.195:44799
10:44:31.859 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

10:44:31.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:31.861374 20627 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:31.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/logs
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.195:44799
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.195
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44795
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.195
10:44:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:44:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:32.185159 24592 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
10:44:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:32.185761 24592 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:44:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:32.186435 24592 flags.cc:432] Enabled unsafe flag: --never_fsync=true
10:44:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:32.228577 24592 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
10:44:32.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:32.229583 24592 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.36.195
10:44:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:32.282202 24592 tablet_server_runner.cc:78] Tablet server non-default flags:
10:44:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:44:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.36.212:38227
10:44:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:44:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:44:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data
10:44:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.36.195:44799
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.36.195
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44795
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.36.254:38609,127.20.36.253:37673,127.20.36.252:35339
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.36.195
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/logs
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 14880490066cc6e355a232e5a6411307cd134162
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 10:27:49 UTC on e7f111948823
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17618
10:44:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:44:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:32.283960 24592 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:44:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:32.286104 24592 file_cache.cc:504] Constructed file cache file cache with capacity 419430
10:44:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:32.305226 24609 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:33.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:32.306883 24610 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:33.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:33.611557 24614 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:44:33.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:33.617597 24613 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1310 milliseconds
10:44:33.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.617728 24592 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:44:33.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.619238 24592 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:44:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.622462 24592 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:44:33.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.623904 24592 hybrid_clock.cc:648] HybridClock initialized: now 1786445073623858 us; error 43 us; skew 500 ppm
10:44:33.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.624997 24592 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:44:33.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.631964 24592 webserver.cc:533] Webserver started at http://127.20.36.195:44795/ using document root <none> and password file <none>
10:44:33.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.632895 24592 fs_manager.cc:362] Metadata directory not provided
10:44:33.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.633203 24592 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:44:33.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.641572 24592 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.000s	sys 0.004s
10:44:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.647192 24634 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
10:44:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 10:44:33.647303 24064 consensus_peers.cc:597] T d339d350474b44a98222592091597860 P cdd21cb83af541e895b0058bf4e0b6d2 -> Peer a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799): Couldn't send request to peer a766ec61926c4e27b86cde01c49ea7ed. Status: Network error: Client connection negotiation failed: client connection to 127.20.36.195:44799: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
10:44:33.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.648540 24592 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
10:44:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.648833 24592 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data,/tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a766ec61926c4e27b86cde01c49ea7ed"
10:44:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 10:43:42 on dist-test-slave-gw5c"
10:44:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.651250 24592 fs_report.cc:389] FS layout report
10:44:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:44:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/wal
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/data
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:44:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:44:33.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.711944 24592 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
10:44:33.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.713548 24592 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:44:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.713959 24592 kserver.cc:163] Server-wide thread pool size limit: 3276
10:44:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.716750 24592 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:44:33.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.722828 24645 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
10:44:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.731333 24592 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
10:44:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.731665 24592 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.001s
10:44:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.731885 24592 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
10:44:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.736716 24592 ts_tablet_manager.cc:616] Registered 1 tablets
10:44:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.736903 24592 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
10:44:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.737504 24645 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap starting.
10:44:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.824632 24645 log.cc:826] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Log is configured to *not* fsync() on all Append() calls
10:44:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.939066 24592 rpc_server.cc:307] RPC server started. Bound to: 127.20.36.195:44799
10:44:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.939185 24808 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.36.195:44799 every 8 connection(s)
10:44:33.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.942430 24592 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskA_7F9h/test-tmp/mini-kudu-cluster15525369760478200045/ts-2/data/info.pb
10:44:33.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.944025 20627 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu as pid 24592
10:44:33.944 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
10:44:33.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.947661 24645 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:44:33.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.948801 24645 tablet_bootstrap.cc:492] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Bootstrap complete.
10:44:33.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.950933 24645 ts_tablet_manager.cc:1403] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Time spent bootstrapping tablet: real 0.214s	user 0.152s	sys 0.053s
10:44:33.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.975242 24645 raft_consensus.cc:359] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:33.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.979116 24645 raft_consensus.cc:740] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: a766ec61926c4e27b86cde01c49ea7ed, State: Initialized, Role: FOLLOWER
10:44:33.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.980890 24645 consensus_queue.cc:260] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a766ec61926c4e27b86cde01c49ea7ed" member_type: VOTER last_known_addr { host: "127.20.36.195" port: 44799 } } peers { permanent_uuid: "c5373a0a37e14c38bc09e40a3616b123" member_type: VOTER last_known_addr { host: "127.20.36.194" port: 39729 } } peers { permanent_uuid: "cdd21cb83af541e895b0058bf4e0b6d2" member_type: VOTER last_known_addr { host: "127.20.36.193" port: 34951 } }
10:44:33.984 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:44:33.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:33.995899 24645 ts_tablet_manager.cc:1434] T d339d350474b44a98222592091597860 P a766ec61926c4e27b86cde01c49ea7ed: Time spent starting tablet: real 0.045s	user 0.027s	sys 0.007s
10:44:33.999 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786445023379, tablet=d339d350474b44a98222592091597860, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
10:44:34.007 [DEBUG - main] (Connection.java:575) [peer master-127.20.36.253:37673(127.20.36.253:37673)] disconnecting while in state READY
10:44:34.008 [DEBUG - main] (Connection.java:575) [peer master-127.20.36.252:35339(127.20.36.252:35339)] disconnecting while in state READY
10:44:34.008 [DEBUG - main] (Connection.java:575) [peer a766ec61926c4e27b86cde01c49ea7ed(127.20.36.195:44799)] disconnecting while in state TERMINATED
10:44:34.009 [DEBUG - main] (Connection.java:575) [peer c5373a0a37e14c38bc09e40a3616b123(127.20.36.194:39729)] disconnecting while in state TERMINATED
10:44:34.010 [DEBUG - main] (Connection.java:575) [peer master-127.20.36.254:38609(127.20.36.254:38609)] disconnecting while in state READY
10:44:34.010 [DEBUG - main] (Connection.java:575) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] disconnecting while in state READY
10:44:34.010 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.36.253:37673(127.20.36.253:37673)] handling channelInactive
10:44:34.011 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.20.36.254:38609(127.20.36.254:38609)] handling channelInactive
10:44:34.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.36.253:37673(127.20.36.253:37673)] cleaning up while in state READY due to: connection closed
10:44:34.011 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.20.36.254:38609(127.20.36.254:38609)] cleaning up while in state READY due to: connection closed
10:44:34.011 [DEBUG - kudu-nio-4] (Connection.java:239) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] handling channelInactive
10:44:34.011 [DEBUG - kudu-nio-4] (Connection.java:709) [peer cdd21cb83af541e895b0058bf4e0b6d2(127.20.36.193:34951)] cleaning up while in state READY due to: connection closed
10:44:34.013 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
10:44:34.014 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.36.252:35339(127.20.36.252:35339)] handling channelInactive
10:44:34.015 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.36.252:35339(127.20.36.252:35339)] cleaning up while in state READY due to: connection closed
10:44:34.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.015036 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 24026
10:44:34.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.037943 24812 heartbeater.cc:344] Connected to a master server at 127.20.36.252:35339
10:44:34.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.041435 24812 heartbeater.cc:461] Registering TS with master...
10:44:34.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.042938 24812 heartbeater.cc:507] Master 127.20.36.252:35339 requested a full tablet report, sending...
10:44:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.048166 20948 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:44:34.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.053122 24813 heartbeater.cc:344] Connected to a master server at 127.20.36.253:37673
10:44:34.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.053488 24813 heartbeater.cc:461] Registering TS with master...
10:44:34.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.054380 24813 heartbeater.cc:507] Master 127.20.36.253:37673 requested a full tablet report, sending...
10:44:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.057971 20824 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:44:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.058445 24823 heartbeater.cc:344] Connected to a master server at 127.20.36.254:38609
10:44:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.058779 24823 heartbeater.cc:461] Registering TS with master...
10:44:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.059839 24823 heartbeater.cc:507] Master 127.20.36.254:38609 requested a full tablet report, sending...
10:44:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.062754 20699 ts_manager.cc:194] Re-registered known tserver with Master: a766ec61926c4e27b86cde01c49ea7ed (127.20.36.195:44799)
10:44:34.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.065969 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.36.195:34119
10:44:34.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.071373 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 24316
10:44:34.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.110893 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 24592
10:44:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.145521 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 20645
10:44:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.188510 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 20768
10:44:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 10:44:34.230319 20627 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskA_7F9h/build/tsan/bin/kudu with pid 20883
10:44:35.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T10:44:35Z chronyd exiting

Time: 69.511

OK (1 test)

