JUnit version 4.13.2
.08:03:34.093 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1779091414085
08:03:34.142 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
08:03:34.196 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:03:34.207 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskjmZNRn/test-tmp
08:03:34.213 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:03:34.216 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskjmZNRn/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:03:34.400 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

08:03:34.599 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
08:03:34.601 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

08:03:34.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.605585 21137 mini_oidc.cc:150] Starting JWKS server
08:03:34.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.609339 21137 webserver.cc:533] Webserver started at http://127.0.0.1:45889/ using document root <none> and password file <none>
08:03:34.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.609900 21137 mini_oidc.cc:186] Starting OIDC Discovery server
08:03:34.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.612061 21137 webserver.cc:533] Webserver started at http://127.0.0.1:33151/ using document root <none> and password file <none>
08:03:34.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T08:03:34Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:03:34.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T08:03:34Z Disabled control of system clock
08:03:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.670748 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/wal
08:03:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/data
08:03:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/logs
08:03:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/data/info.pb
08:03:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:34.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:34.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:34.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:34.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:34.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:34.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:34.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:34.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:34.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.126:37373
08:03:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.126
08:03:34.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:34.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:34.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45889/jwks.json with env {}
08:03:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:34.967242 21150 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:34.967908 21150 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:34.968669 21150 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:35.001412 21150 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:35.001789 21150 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:35.002143 21150 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:35.002480 21150 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:35.002837 21150 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:35.003125 21150 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45889/jwks.json
08:03:35.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:35.039283 21150 master_runner.cc:387] Master server non-default flags:
08:03:35.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:35.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:35.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:35.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:35.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/data
08:03:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/wal
08:03:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.126:37373
08:03:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45889/jwks.json
08:03:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/data/info.pb
08:03:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.126
08:03:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/logs
08:03:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:03:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:03:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:35.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:35.041074 21150 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:35.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:35.043316 21150 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:35.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:35.056588 21156 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:36.460455 21155 debug-util.cc:398] Leaking SignalData structure 0x7b08000271e0 after lost signal to thread 21150
08:03:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:36.850744 21155 kernel_stack_watchdog.cc:198] Thread 21150 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
08:03:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:03:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:03:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:03:36.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:03:36.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:35.057691 21157 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:36.853042 21150 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.796s	user 0.626s	sys 1.171s
08:03:36.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:36.853552 21150 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.797s	user 0.626s	sys 1.171s
08:03:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:36.855643 21159 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:36.857733 21158 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1796 milliseconds
08:03:36.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.857812 21150 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:36.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.859035 21150 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:36.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.861620 21150 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.862949 21150 hybrid_clock.cc:648] HybridClock initialized: now 1779091416862925 us; error 56 us; skew 500 ppm
08:03:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.863837 21150 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:36.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.869594 21150 webserver.cc:533] Webserver started at http://127.20.164.126:42671/ using document root <none> and password file <none>
08:03:36.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.870451 21150 fs_manager.cc:362] Metadata directory not provided
08:03:36.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.870668 21150 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:36.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.871173 21150 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:36.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.875625 21150 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/data/instance:
08:03:36.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6f04fd360594de08eb1fe8d209fac78"
08:03:36.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:36 on dist-test-slave-0rvw"
08:03:36.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.876601 21150 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/wal/instance:
08:03:36.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6f04fd360594de08eb1fe8d209fac78"
08:03:36.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:36 on dist-test-slave-0rvw"
08:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.885370 21150 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.006s	sys 0.003s
08:03:36.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.891057 21166 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:36.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.892419 21150 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
08:03:36.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.892808 21150 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/wal
08:03:36.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6f04fd360594de08eb1fe8d209fac78"
08:03:36.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:36 on dist-test-slave-0rvw"
08:03:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.893270 21150 fs_report.cc:389] FS layout report
08:03:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/wal
08:03:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/wal
08:03:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/data/data
08:03:36.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:36.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:36.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:36.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:36.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:36.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:36.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:36.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:36.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:36.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:36.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.949647 21150 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:36.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.951653 21150 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:36.952205 21150 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.023722 21150 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.126:37373
08:03:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.023798 21217 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.126:37373 every 8 connection(s)
08:03:37.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.026700 21150 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/data/info.pb
08:03:37.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.032483 21218 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:37.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.037441 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 21150
08:03:37.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.037957 21137 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-0/wal/instance
08:03:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.043242 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/wal
08:03:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/data
08:03:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:37.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:37.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:37.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:37.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:37.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/logs
08:03:37.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/data/info.pb
08:03:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:37.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:37.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:37.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:37.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:37.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:37.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:37.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.125:40721
08:03:37.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.125
08:03:37.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:37.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:37.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:37.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:37.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:37.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:37.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:37.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:37.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45889/jwks.json with env {}
08:03:37.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.046645 21218 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } has no permanent_uuid. Determining permanent_uuid...
08:03:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.077877 21218 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } has no permanent_uuid. Determining permanent_uuid...
08:03:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.080865 21168 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.125:40721: connect: Connection refused (error 111)
08:03:37.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.085000 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.125:40721: Network error: Client connection negotiation failed: client connection to 127.20.164.125:40721: connect: Connection refused (error 111)
08:03:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.134491 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } attempt: 1
08:03:37.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.138978 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.125:40721: Network error: Client connection negotiation failed: client connection to 127.20.164.125:40721: connect: Connection refused (error 111)
08:03:37.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.207435 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } attempt: 2
08:03:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.211242 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.125:40721: Network error: Client connection negotiation failed: client connection to 127.20.164.125:40721: connect: Connection refused (error 111)
08:03:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.302706 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } attempt: 3
08:03:37.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.307487 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.125:40721: Network error: Client connection negotiation failed: client connection to 127.20.164.125:40721: connect: Connection refused (error 111)
08:03:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.359660 21220 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.360149 21220 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.360725 21220 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:37.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.392226 21220 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:37.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.392489 21220 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:37.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.392690 21220 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:37.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.392861 21220 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:37.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.393030 21220 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:37.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.393263 21220 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45889/jwks.json
08:03:37.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.428390 21220 master_runner.cc:387] Master server non-default flags:
08:03:37.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:37.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:37.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:37.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:37.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/data
08:03:37.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/wal
08:03:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.125:40721
08:03:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45889/jwks.json
08:03:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/data/info.pb
08:03:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.125
08:03:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/logs
08:03:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:03:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:03:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.429951 21220 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.431995 21220 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:37.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.445079 21229 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:37.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.450917 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } attempt: 4
08:03:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.458703 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.125:40721: Network error: Client connection negotiation failed: client connection to 127.20.164.125:40721: connect: Connection refused (error 111)
08:03:37.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:37.758183 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } attempt: 5
08:03:37.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.767220 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.125:40721: Network error: Client connection negotiation failed: client connection to 127.20.164.125:40721: connect: Connection refused (error 111)
08:03:38.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.314805 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } attempt: 6
08:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.347805 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.125:40721: Network error: Client connection negotiation failed: client connection to 127.20.164.125:40721: connect: Connection refused (error 111)
08:03:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.447018 21230 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.806161 21231 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1355 milliseconds
08:03:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.807554 21232 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.807614 21220 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.362s	user 0.417s	sys 0.927s
08:03:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.808012 21220 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.363s	user 0.417s	sys 0.927s
08:03:38.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.808308 21220 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.811801 21220 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:38.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.815140 21220 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:38.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.816861 21220 hybrid_clock.cc:648] HybridClock initialized: now 1779091418816788 us; error 76 us; skew 500 ppm
08:03:38.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.817683 21220 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:38.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.825546 21220 webserver.cc:533] Webserver started at http://127.20.164.125:37939/ using document root <none> and password file <none>
08:03:38.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.826364 21220 fs_manager.cc:362] Metadata directory not provided
08:03:38.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.826550 21220 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:38.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.826968 21220 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:38.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.831776 21220 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/data/instance:
08:03:38.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "244122b0c4f441a0a5afbf256b34cce9"
08:03:38.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:38 on dist-test-slave-0rvw"
08:03:38.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.832788 21220 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/wal/instance:
08:03:38.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "244122b0c4f441a0a5afbf256b34cce9"
08:03:38.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:38 on dist-test-slave-0rvw"
08:03:38.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.840080 21220 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.000s
08:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.846354 21241 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.847913 21220 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.001s	sys 0.001s
08:03:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.848282 21220 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/wal
08:03:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "244122b0c4f441a0a5afbf256b34cce9"
08:03:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:38 on dist-test-slave-0rvw"
08:03:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.848666 21220 fs_report.cc:389] FS layout report
08:03:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/wal
08:03:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/wal
08:03:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/data/data
08:03:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:38.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.922552 21220 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:38.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.924211 21220 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:38.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.924705 21220 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.000600 21220 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.125:40721
08:03:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.000681 21292 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.125:40721 every 8 connection(s)
08:03:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.003684 21220 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/data/info.pb
08:03:39.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.006336 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 21220
08:03:39.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.006819 21137 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-1/wal/instance
08:03:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.009480 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/wal
08:03:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/data
08:03:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/logs
08:03:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/data/info.pb
08:03:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:39.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.124:42449
08:03:39.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.124
08:03:39.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:39.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45889/jwks.json with env {}
08:03:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.010756 21293 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.026643 21293 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } has no permanent_uuid. Determining permanent_uuid...
08:03:39.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.061031 21293 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } has no permanent_uuid. Determining permanent_uuid...
08:03:39.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.076429 21293 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } has no permanent_uuid. Determining permanent_uuid...
08:03:39.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.078662 21244 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.082682 21293 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.132121 21293 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 1
08:03:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.136559 21293 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.204958 21293 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 2
08:03:39.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.209141 21293 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.300601 21293 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 3
08:03:39.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.304342 21293 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.343734 21295 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:39.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.344305 21295 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:39.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.344745 21295 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:39.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.375643 21295 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:39.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.375892 21295 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:39.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.376078 21295 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:39.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.376266 21295 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:39.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.376574 21295 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:39.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.376804 21295 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45889/jwks.json
08:03:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.408368 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } attempt: 7
08:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.413552 21295 master_runner.cc:387] Master server non-default flags:
08:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/data
08:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/wal
08:03:39.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:39.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:39.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:39.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:39.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:39.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.124:42449
08:03:39.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:39.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:39.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:39.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45889/jwks.json
08:03:39.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:39.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:39.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/data/info.pb
08:03:39.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.124
08:03:39.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:39.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:39.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:39.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/logs
08:03:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:03:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:03:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.415297 21295 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.417572 21295 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:39.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.421994 21218 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } has no permanent_uuid. Determining permanent_uuid...
08:03:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.428440 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.431854 21307 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.447693 21293 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 4
08:03:39.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.451861 21293 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.486888 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 1
08:03:39.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.490789 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.572160 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 2
08:03:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.577253 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.662696 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 3
08:03:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.667695 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.751448 21293 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 5
08:03:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.755658 21293 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:39.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.808086 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 4
08:03:39.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.813078 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:40.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.096526 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 5
08:03:40.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.101322 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.303220 21293 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 6
08:03:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.311259 21293 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.432816 21308 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:40.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.632269 21309 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:03:40.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.633715 21310 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:40.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.633711 21295 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:40.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.637153 21295 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:40.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.640139 21295 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:40.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.641537 21295 hybrid_clock.cc:648] HybridClock initialized: now 1779091420641492 us; error 52 us; skew 500 ppm
08:03:40.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.642277 21295 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.649082 21295 webserver.cc:533] Webserver started at http://127.20.164.124:36851/ using document root <none> and password file <none>
08:03:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.649917 21295 fs_manager.cc:362] Metadata directory not provided
08:03:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.650120 21295 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.650488 21295 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:40.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.653766 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 6
08:03:40.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.655416 21295 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/data/instance:
08:03:40.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "895f900bc5104e43a82218ecafa1a26d"
08:03:40.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:40 on dist-test-slave-0rvw"
08:03:40.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.656941 21295 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/wal/instance:
08:03:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "895f900bc5104e43a82218ecafa1a26d"
08:03:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:40 on dist-test-slave-0rvw"
08:03:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.660284 21218 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.164.124:42449: Network error: Client connection negotiation failed: client connection to 127.20.164.124:42449: connect: Connection refused (error 111)
08:03:40.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.665033 21295 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.002s	sys 0.006s
08:03:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.670624 21321 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.671748 21295 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
08:03:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.672030 21295 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/wal
08:03:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "895f900bc5104e43a82218ecafa1a26d"
08:03:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:40 on dist-test-slave-0rvw"
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.672415 21295 fs_report.cc:389] FS layout report
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/wal
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/wal
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/data/data
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:40.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:40.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.723711 21295 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.725075 21295 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.725462 21295 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:40.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.799618 21295 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.124:42449
08:03:40.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.799690 21372 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.124:42449 every 8 connection(s)
08:03:40.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.802249 21295 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/data/info.pb
08:03:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.809072 21373 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:40.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.811709 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 21295
08:03:40.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.812170 21137 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/master-2/wal/instance
08:03:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.825618 21373 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } has no permanent_uuid. Determining permanent_uuid...
08:03:40.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.852813 21373 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } has no permanent_uuid. Determining permanent_uuid...
08:03:40.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.863636 21373 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } has no permanent_uuid. Determining permanent_uuid...
08:03:40.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.879483 21373 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d: Bootstrap starting.
08:03:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.886924 21373 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d: Neither blocks nor log segments found. Creating new log.
08:03:40.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.889097 21373 log.cc:826] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d: Log is configured to *not* fsync() on all Append() calls
08:03:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.893918 21373 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d: No bootstrap required, opened a new log
08:03:40.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.913842 21373 raft_consensus.cc:359] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } }
08:03:40.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.914422 21373 raft_consensus.cc:385] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:40.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.914716 21373 raft_consensus.cc:740] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 895f900bc5104e43a82218ecafa1a26d, State: Initialized, Role: FOLLOWER
08:03:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.915421 21373 consensus_queue.cc:260] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } }
08:03:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.917831 21382 sys_catalog.cc:455] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } } }
08:03:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.918519 21382 sys_catalog.cc:458] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [sys.catalog]: This master's current role is: FOLLOWER
08:03:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.919420 21373 sys_catalog.cc:565] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [sys.catalog]: configured and running, proceeding with master startup.
08:03:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.946139 21393 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.946487 21393 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.941195 21295 master_runner.cc:187] Error getting master registration for 127.20.164.126:37373: OK, instance_id { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" instance_seqno: 1779091417004510 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.972054 21295 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:03:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.980813 21327 tablet.cc:2404] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.083851 21382 raft_consensus.cc:493] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:41.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.084347 21382 raft_consensus.cc:515] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } }
08:03:41.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.089018 21382 leader_election.cc:290] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f6f04fd360594de08eb1fe8d209fac78 (127.20.164.126:37373), 244122b0c4f441a0a5afbf256b34cce9 (127.20.164.125:40721)
08:03:41.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.371646 21293 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 7
08:03:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.394155 21293 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9: Bootstrap starting.
08:03:41.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.405123 21293 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9: Neither blocks nor log segments found. Creating new log.
08:03:41.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.407306 21293 log.cc:826] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9: Log is configured to *not* fsync() on all Append() calls
08:03:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.414201 21293 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9: No bootstrap required, opened a new log
08:03:41.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.441897 21293 raft_consensus.cc:359] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } }
08:03:41.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.442730 21293 raft_consensus.cc:385] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:41.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.442997 21293 raft_consensus.cc:740] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 244122b0c4f441a0a5afbf256b34cce9, State: Initialized, Role: FOLLOWER
08:03:41.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.444569 21293 consensus_queue.cc:260] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } }
08:03:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.452594 21398 sys_catalog.cc:455] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } } }
08:03:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.453368 21398 sys_catalog.cc:458] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [sys.catalog]: This master's current role is: FOLLOWER
08:03:41.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.456543 21293 sys_catalog.cc:565] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [sys.catalog]: configured and running, proceeding with master startup.
08:03:41.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.456736 21268 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "895f900bc5104e43a82218ecafa1a26d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "244122b0c4f441a0a5afbf256b34cce9" is_pre_election: true
08:03:41.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.457638 21268 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 895f900bc5104e43a82218ecafa1a26d in term 0.
08:03:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.459131 21323 leader_election.cc:304] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 244122b0c4f441a0a5afbf256b34cce9, 895f900bc5104e43a82218ecafa1a26d; no voters: 
08:03:41.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.460927 21382 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:41.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.461263 21382 raft_consensus.cc:493] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:41.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.463667 21382 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 0 FOLLOWER]: Advancing to term 1
08:03:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.470723 21382 raft_consensus.cc:515] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } }
08:03:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.482339 21268 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "895f900bc5104e43a82218ecafa1a26d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "244122b0c4f441a0a5afbf256b34cce9"
08:03:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.482887 21268 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [term 0 FOLLOWER]: Advancing to term 1
08:03:41.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.490450 21268 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 895f900bc5104e43a82218ecafa1a26d in term 1.
08:03:41.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.491619 21323 leader_election.cc:304] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 244122b0c4f441a0a5afbf256b34cce9, 895f900bc5104e43a82218ecafa1a26d; no voters: 
08:03:41.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.496961 21382 leader_election.cc:290] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [CANDIDATE]: Term 1 election: Requested vote from peers f6f04fd360594de08eb1fe8d209fac78 (127.20.164.126:37373), 244122b0c4f441a0a5afbf256b34cce9 (127.20.164.125:40721)
08:03:41.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.497534 21382 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 1 FOLLOWER]: Leader election won for term 1
08:03:41.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.498009 21382 raft_consensus.cc:697] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [term 1 LEADER]: Becoming Leader. State: Replica: 895f900bc5104e43a82218ecafa1a26d, State: Running, Role: LEADER
08:03:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.498996 21382 consensus_queue.cc:237] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } }
08:03:41.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.517871 21406 sys_catalog.cc:455] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 895f900bc5104e43a82218ecafa1a26d. Latest consensus state: current_term: 1 leader_uuid: "895f900bc5104e43a82218ecafa1a26d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } } }
08:03:41.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.519181 21406 sys_catalog.cc:458] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [sys.catalog]: This master's current role is: LEADER
08:03:41.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.522428 21410 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
08:03:41.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.530177 21410 catalog_manager.cc:1519] Initializing Kudu cluster ID...
08:03:41.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.558761 21268 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [term 1 FOLLOWER]: Refusing update from remote peer 895f900bc5104e43a82218ecafa1a26d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.560407 21406 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [LEADER]: Connected to new peer: Peer: permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.541159 21220 master_runner.cc:187] Error getting master registration for 127.20.164.126:37373: OK, instance_id { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" instance_seqno: 1779091417004510 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:41.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.570359 21220 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:03:41.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:41.573299 21412 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:41.573632 21412 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.610080 21398 sys_catalog.cc:455] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 895f900bc5104e43a82218ecafa1a26d. Latest consensus state: current_term: 1 leader_uuid: "895f900bc5104e43a82218ecafa1a26d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } } }
08:03:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.611536 21398 sys_catalog.cc:458] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [sys.catalog]: This master's current role is: FOLLOWER
08:03:41.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.617157 21413 mvcc.cc:204] Tried to move back new op lower bound from 7287158462665052160 to 7287158462481915904. Current Snapshot: MvccSnapshot[applied={T|T < 7287158462665052160}]
08:03:41.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.628355 21398 sys_catalog.cc:455] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "895f900bc5104e43a82218ecafa1a26d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } } }
08:03:41.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.629449 21398 sys_catalog.cc:458] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9 [sys.catalog]: This master's current role is: FOLLOWER
08:03:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.634886 21406 sys_catalog.cc:455] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "895f900bc5104e43a82218ecafa1a26d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } } }
08:03:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.635593 21406 sys_catalog.cc:458] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [sys.catalog]: This master's current role is: LEADER
08:03:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.636456 21410 catalog_manager.cc:1382] Generated new cluster ID: 2ab7c48cf4d64bb3a7080d0c029d52ec
08:03:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.636718 21410 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
08:03:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.685128 21410 catalog_manager.cc:1405] Generated new certificate authority record
08:03:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.693646 21218 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } attempt: 7
08:03:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.694519 21410 catalog_manager.cc:1539] Loading token signing keys...
08:03:41.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.727181 21218 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78: Bootstrap starting.
08:03:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.728715 21410 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d: Generated new TSK 0
08:03:41.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.729779 21410 catalog_manager.cc:1549] Initializing in-progress tserver states...
08:03:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.742410 21218 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78: Neither blocks nor log segments found. Creating new log.
08:03:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.744553 21218 log.cc:826] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78: Log is configured to *not* fsync() on all Append() calls
08:03:41.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.753103 21218 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78: No bootstrap required, opened a new log
08:03:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.781446 21218 raft_consensus.cc:359] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } }
08:03:41.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.782617 21218 raft_consensus.cc:385] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:41.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.783248 21218 raft_consensus.cc:740] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f6f04fd360594de08eb1fe8d209fac78, State: Initialized, Role: FOLLOWER
08:03:41.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.785018 21218 consensus_queue.cc:260] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } }
08:03:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.790814 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:03:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data
08:03:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:41.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:41.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/logs
08:03:41.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:03:41.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:41.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:41.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:41.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.65:0
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.65
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.65
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:41.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.790024 21421 sys_catalog.cc:455] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } } }
08:03:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.791849 21421 sys_catalog.cc:458] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [sys.catalog]: This master's current role is: FOLLOWER
08:03:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.799553 21421 raft_consensus.cc:493] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.800937 21421 raft_consensus.cc:515] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } }
08:03:41.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.807228 21218 sys_catalog.cc:565] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [sys.catalog]: configured and running, proceeding with master startup.
08:03:41.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.807344 21268 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f6f04fd360594de08eb1fe8d209fac78" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "244122b0c4f441a0a5afbf256b34cce9" is_pre_election: true
08:03:41.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.808470 21193 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "895f900bc5104e43a82218ecafa1a26d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f6f04fd360594de08eb1fe8d209fac78" is_pre_election: true
08:03:41.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.809194 21193 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 895f900bc5104e43a82218ecafa1a26d in term 0.
08:03:41.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.810518 21192 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "895f900bc5104e43a82218ecafa1a26d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f6f04fd360594de08eb1fe8d209fac78"
08:03:41.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.811054 21192 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [term 0 FOLLOWER]: Advancing to term 1
08:03:41.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.814845 21421 leader_election.cc:290] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 244122b0c4f441a0a5afbf256b34cce9 (127.20.164.125:40721), 895f900bc5104e43a82218ecafa1a26d (127.20.164.124:42449)
08:03:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.816102 21348 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f6f04fd360594de08eb1fe8d209fac78" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "895f900bc5104e43a82218ecafa1a26d" is_pre_election: true
08:03:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.818133 21192 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 895f900bc5104e43a82218ecafa1a26d in term 1.
08:03:41.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.818512 21169 leader_election.cc:304] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f6f04fd360594de08eb1fe8d209fac78; no voters: 244122b0c4f441a0a5afbf256b34cce9, 895f900bc5104e43a82218ecafa1a26d
08:03:41.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.819661 21191 raft_consensus.cc:1275] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [term 1 FOLLOWER]: Refusing update from remote peer 895f900bc5104e43a82218ecafa1a26d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:41.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.820156 21421 raft_consensus.cc:2749] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:03:41.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.827155 21406 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [LEADER]: Connected to new peer: Peer: permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:41.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.902578 21421 sys_catalog.cc:455] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 895f900bc5104e43a82218ecafa1a26d. Latest consensus state: current_term: 1 leader_uuid: "895f900bc5104e43a82218ecafa1a26d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } } }
08:03:41.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.903303 21421 sys_catalog.cc:458] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [sys.catalog]: This master's current role is: FOLLOWER
08:03:41.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.903126 21430 sys_catalog.cc:455] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "895f900bc5104e43a82218ecafa1a26d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } } }
08:03:41.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.905077 21430 sys_catalog.cc:458] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78 [sys.catalog]: This master's current role is: FOLLOWER
08:03:41.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.923297 21439 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:03:42.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.386588 21406 sys_catalog.cc:455] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "895f900bc5104e43a82218ecafa1a26d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f04fd360594de08eb1fe8d209fac78" member_type: VOTER last_known_addr { host: "127.20.164.126" port: 37373 } } peers { permanent_uuid: "244122b0c4f441a0a5afbf256b34cce9" member_type: VOTER last_known_addr { host: "127.20.164.125" port: 40721 } } peers { permanent_uuid: "895f900bc5104e43a82218ecafa1a26d" member_type: VOTER last_known_addr { host: "127.20.164.124" port: 42449 } } }
08:03:42.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.387333 21406 sys_catalog.cc:458] T 00000000000000000000000000000000 P 895f900bc5104e43a82218ecafa1a26d [sys.catalog]: This master's current role is: LEADER
08:03:42.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.392628 21422 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:42.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.393939 21422 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:42.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.394953 21422 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.459102 21422 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:42.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.460561 21422 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.65
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.528442 21422 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.65:0
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:03:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.65
08:03:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.65
08:03:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/logs
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.533145 21422 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.535264 21422 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:42.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.551612 21448 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.586133 21412 catalog_manager.cc:1294] Loaded cluster ID: 2ab7c48cf4d64bb3a7080d0c029d52ec
08:03:42.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.587046 21412 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9: loading cluster ID for follower catalog manager: success
08:03:42.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.596932 21412 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9: acquiring CA information for follower catalog manager: success
08:03:42.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.614187 21412 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 244122b0c4f441a0a5afbf256b34cce9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:42.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.944226 21439 catalog_manager.cc:1294] Loaded cluster ID: 2ab7c48cf4d64bb3a7080d0c029d52ec
08:03:42.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.945035 21439 catalog_manager.cc:1587] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78: loading cluster ID for follower catalog manager: success
08:03:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.953842 21439 catalog_manager.cc:1609] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78: acquiring CA information for follower catalog manager: success
08:03:42.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.973591 21439 catalog_manager.cc:1637] T 00000000000000000000000000000000 P f6f04fd360594de08eb1fe8d209fac78: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:43.956055 21447 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 21422
08:03:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.410019 21447 kernel_stack_watchdog.cc:198] Thread 21422 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
08:03:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:03:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:03:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:03:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:03:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.554682 21449 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:44.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.413293 21451 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:44.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.413916 21422 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.861s	user 0.417s	sys 0.651s
08:03:44.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.414245 21422 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.861s	user 0.417s	sys 0.651s
08:03:44.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.418977 21450 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1865 milliseconds
08:03:44.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.419093 21422 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:44.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.420892 21422 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:44.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.426698 21422 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.428598 21422 hybrid_clock.cc:648] HybridClock initialized: now 1779091424428568 us; error 112 us; skew 500 ppm
08:03:44.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.429768 21422 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:44.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.459791 21422 webserver.cc:533] Webserver started at http://127.20.164.65:33865/ using document root <none> and password file <none>
08:03:44.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.461351 21422 fs_manager.cc:362] Metadata directory not provided
08:03:44.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.461836 21422 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:44.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.462548 21422 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:44.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.469458 21422 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/instance:
08:03:44.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e8aea6cb22324b45b188c416f3d49634"
08:03:44.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:44 on dist-test-slave-0rvw"
08:03:44.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.471696 21422 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal/instance:
08:03:44.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e8aea6cb22324b45b188c416f3d49634"
08:03:44.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:44 on dist-test-slave-0rvw"
08:03:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.486886 21422 fs_manager.cc:696] Time spent creating directory manager: real 0.014s	user 0.004s	sys 0.008s
08:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.502079 21459 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.503979 21422 fs_manager.cc:730] Time spent opening block manager: real 0.011s	user 0.005s	sys 0.005s
08:03:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.504645 21422 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:03:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e8aea6cb22324b45b188c416f3d49634"
08:03:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:44 on dist-test-slave-0rvw"
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.505345 21422 fs_report.cc:389] FS layout report
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/data
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:44.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.641079 21422 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:44.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.643468 21422 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:44.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.644388 21422 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.653785 21422 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:44.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.666266 21422 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:44.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.666920 21422 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.001s	sys 0.000s
08:03:44.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.667476 21422 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:44.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.667877 21422 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:03:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.993736 21422 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.65:46489
08:03:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.993875 21573 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.65:46489 every 8 connection(s)
08:03:44.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.998560 21422 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:03:45.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.008176 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 21422
08:03:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.008790 21137 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal/instance
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.018711 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/logs
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.66:0
08:03:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.66
08:03:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.66
08:03:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:45.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.096621 21574 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:03:45.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.097147 21574 heartbeater.cc:461] Registering TS with master...
08:03:45.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.098498 21574 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:03:45.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.101457 21338 ts_manager.cc:194] Registered new tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:03:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.107213 21338 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.65:38829
08:03:45.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.123983 21576 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:03:45.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.124351 21576 heartbeater.cc:461] Registering TS with master...
08:03:45.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.125038 21576 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:03:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.128067 21258 ts_manager.cc:194] Registered new tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:03:45.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.152522 21579 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:03:45.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.152947 21579 heartbeater.cc:461] Registering TS with master...
08:03:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.153705 21579 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:03:45.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.157927 21183 ts_manager.cc:194] Registered new tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:03:45.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:45.405160 21575 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:45.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:45.405714 21575 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:45.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:45.406332 21575 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:45.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:45.442674 21575 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:45.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:45.444420 21575 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.66
08:03:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.482976 21575 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data
08:03:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.66:0
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.66
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.66
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/logs
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:03:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:03:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.484764 21575 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.486501 21575 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:45.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:45.500233 21589 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:46.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.112247 21574 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:03:46.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:46.228281 21570 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad060 after lost signal to thread 21440
08:03:46.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:46.229333 21570 debug-util.cc:398] Leaking SignalData structure 0x7b08000adf40 after lost signal to thread 21573
08:03:47.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:46.908329 21588 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 21575
08:03:47.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:45.501581 21590 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:47.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:47.386708 21575 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.885s	user 0.469s	sys 0.940s
08:03:47.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:47.387059 21575 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.885s	user 0.469s	sys 0.940s
08:03:47.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:47.393996 21591 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1885 milliseconds
08:03:47.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:47.395293 21592 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:47.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.395319 21575 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:47.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.396915 21575 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:47.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.399714 21575 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:47.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.401345 21575 hybrid_clock.cc:648] HybridClock initialized: now 1779091427401279 us; error 49 us; skew 500 ppm
08:03:47.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.402549 21575 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:47.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.410313 21575 webserver.cc:533] Webserver started at http://127.20.164.66:35829/ using document root <none> and password file <none>
08:03:47.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.411538 21575 fs_manager.cc:362] Metadata directory not provided
08:03:47.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.411805 21575 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:47.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.412379 21575 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:47.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.419344 21575 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/instance:
08:03:47.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e269425272f249e496b7c12b18f63076"
08:03:47.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:47 on dist-test-slave-0rvw"
08:03:47.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.420845 21575 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal/instance:
08:03:47.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e269425272f249e496b7c12b18f63076"
08:03:47.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:47 on dist-test-slave-0rvw"
08:03:47.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.430145 21575 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.009s	sys 0.001s
08:03:47.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.435741 21602 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.436825 21575 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
08:03:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.437176 21575 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:03:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e269425272f249e496b7c12b18f63076"
08:03:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:47 on dist-test-slave-0rvw"
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.437490 21575 fs_report.cc:389] FS layout report
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/data
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:47.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.490114 21575 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.491560 21575 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:47.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.491930 21575 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:47.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.494522 21575 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:47.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.498837 21575 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:47.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.499056 21575 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
08:03:47.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.499382 21575 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:47.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.499581 21575 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:03:47.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.664888 21575 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.66:39351
08:03:47.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.665057 21714 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.66:39351 every 8 connection(s)
08:03:47.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.667565 21575 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:03:47.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.678129 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 21575
08:03:47.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.678573 21137 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal/instance
08:03:47.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.714435 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/logs
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.67:0
08:03:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.67
08:03:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.67
08:03:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:47.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:47.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:47.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:47.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.721002 21715 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:03:47.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.721513 21715 heartbeater.cc:461] Registering TS with master...
08:03:47.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.722750 21715 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:03:47.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.725786 21338 ts_manager.cc:194] Registered new tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:03:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.727283 21338 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.66:34775
08:03:47.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.728349 21717 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:03:47.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.728685 21717 heartbeater.cc:461] Registering TS with master...
08:03:47.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.729503 21717 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:03:47.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.730391 21716 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:03:47.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.731253 21716 heartbeater.cc:461] Registering TS with master...
08:03:47.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.731949 21183 ts_manager.cc:194] Registered new tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:03:47.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.731933 21716 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:47.735405 21258 ts_manager.cc:194] Registered new tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:03:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:48.090901 21725 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:48.091471 21725 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:48.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:48.092042 21725 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:48.126767 21725 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:48.127688 21725 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.67
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:48.164906 21725 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.67:0
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.67
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.67
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/logs
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:03:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:03:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:48.167038 21725 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:48.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:48.168730 21725 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:48.182848 21731 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:48.731451 21715 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:03:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:48.183405 21732 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.584793 21734 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.587720 21733 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1398 milliseconds
08:03:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.587968 21725 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.404s	user 0.348s	sys 0.924s
08:03:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.588317 21725 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.405s	user 0.348s	sys 0.924s
08:03:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.588600 21725 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:49.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.590085 21725 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:49.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.592633 21725 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:49.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.594180 21725 hybrid_clock.cc:648] HybridClock initialized: now 1779091429594139 us; error 36 us; skew 500 ppm
08:03:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.595363 21725 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:49.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.604136 21725 webserver.cc:533] Webserver started at http://127.20.164.67:42487/ using document root <none> and password file <none>
08:03:49.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.605489 21725 fs_manager.cc:362] Metadata directory not provided
08:03:49.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.605779 21725 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:49.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.606416 21725 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.613759 21725 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/instance:
08:03:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d31321473b59420090c29f4de9f6f928"
08:03:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:49 on dist-test-slave-0rvw"
08:03:49.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.615351 21725 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal/instance:
08:03:49.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d31321473b59420090c29f4de9f6f928"
08:03:49.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:49 on dist-test-slave-0rvw"
08:03:49.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.624583 21725 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.008s	sys 0.000s
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.632279 21742 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.633436 21725 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.633725 21725 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d31321473b59420090c29f4de9f6f928"
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:49 on dist-test-slave-0rvw"
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.634075 21725 fs_report.cc:389] FS layout report
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/data
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:49.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.702489 21725 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.704133 21725 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.704620 21725 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:49.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.707387 21725 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.711694 21725 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.711906 21725 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
08:03:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.712258 21725 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.712445 21725 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:03:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.864573 21725 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.67:37251
08:03:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.864662 21854 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.67:37251 every 8 connection(s)
08:03:49.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.866991 21725 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:03:49.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.872962 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 21725
08:03:49.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.873498 21137 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal/instance
08:03:49.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.927147 21856 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:03:49.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.927663 21856 heartbeater.cc:461] Registering TS with master...
08:03:49.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.928225 21857 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:03:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.928567 21857 heartbeater.cc:461] Registering TS with master...
08:03:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.928957 21856 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:03:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.930086 21855 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:03:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.930063 21857 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:03:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.930500 21855 heartbeater.cc:461] Registering TS with master...
08:03:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.931365 21855 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:03:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.932768 21258 ts_manager.cc:194] Registered new tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:03:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.933506 21183 ts_manager.cc:194] Registered new tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:03:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.933898 21337 ts_manager.cc:194] Registered new tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:03:49.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.935562 21337 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.67:42701
08:03:49.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.942989 21137 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:03:49.954 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
08:03:49.955 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

08:03:49.964 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.20.164.126"
      port: 37373
    }
    bound_http_address {
      host: "127.20.164.126"
      port: 42671
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.20.164.125"
      port: 40721
    }
    bound_http_address {
      host: "127.20.164.125"
      port: 37939
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.20.164.124"
      port: 42449
    }
    bound_http_address {
      host: "127.20.164.124"
      port: 36851
    }
  }
}

08:03:49.966 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

08:03:49.968 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.20.164.65"
      port: 46489
    }
    bound_http_address {
      host: "127.20.164.65"
      port: 33865
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.20.164.66"
      port: 39351
    }
    bound_http_address {
      host: "127.20.164.66"
      port: 35829
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.20.164.67"
      port: 37251
    }
    bound_http_address {
      host: "127.20.164.67"
      port: 42487
    }
  }
}

08:03:49.968 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:03:50.167 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
08:03:50.222 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
08:03:50.365 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.164.126:37373(127.20.164.126:37373)] connecting to peer
08:03:50.413 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.164.124:42449(127.20.164.124:42449)] connecting to peer
08:03:50.428 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.164.125:40721(127.20.164.125:40721)] connecting to peer
08:03:50.429 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.164.126:37373(127.20.164.126:37373)] Successfully connected to peer
08:03:50.434 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.164.124:42449(127.20.164.124:42449)] Successfully connected to peer
08:03:50.441 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.20.164.125:40721(127.20.164.125:40721)] Successfully connected to peer
08:03:50.530 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.164.126
08:03:50.530 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.164.124
08:03:50.535 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.164.125
08:03:50.736 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xe5d76035, L:/127.0.0.1:60024 - R:/127.20.164.125:40721] using SASL/PLAIN
08:03:50.736 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x442821c9, L:/127.0.0.1:58376 - R:/127.20.164.126:37373] using SASL/PLAIN
08:03:50.736 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf8f9b5ba, L:/127.0.0.1:48616 - R:/127.20.164.124:42449] using SASL/PLAIN
08:03:50.775 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:50.776 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=444), Traces: [0ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.164.126:37373, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.164.124:42449, [94ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.164.125:40721, [419ms] Sub RPC ConnectToMaster: received response from server master-127.20.164.126:37373: OK, [419ms] Sub RPC ConnectToMaster: received response from server master-127.20.164.125:40721: OK, [419ms] Sub RPC ConnectToMaster: received response from server master-127.20.164.124:42449: OK, deferred=null) after lookup
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.821336 21337 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:48616:
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318"
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:50.825716 21337 catalog_manager.cc:7058] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:50.903 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 5a6ef159b6014406b067e16f9a854c02
08:03:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.939325 21855 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:03:50.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.957190 21790 tablet_service.cc:1511] Processing CreateTablet for tablet 7ce36047551045458a0528924b6fb281 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318 [id=5a6ef159b6014406b067e16f9a854c02]), partition=RANGE (key) PARTITION UNBOUNDED
08:03:50.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.959136 21790 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7ce36047551045458a0528924b6fb281. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:50.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.977018 21650 tablet_service.cc:1511] Processing CreateTablet for tablet 7ce36047551045458a0528924b6fb281 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318 [id=5a6ef159b6014406b067e16f9a854c02]), partition=RANGE (key) PARTITION UNBOUNDED
08:03:50.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.978760 21650 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7ce36047551045458a0528924b6fb281. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:50.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.990669 21508 tablet_service.cc:1511] Processing CreateTablet for tablet 7ce36047551045458a0528924b6fb281 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318 [id=5a6ef159b6014406b067e16f9a854c02]), partition=RANGE (key) PARTITION UNBOUNDED
08:03:50.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.992435 21508 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7ce36047551045458a0528924b6fb281. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:51.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.011843 21890 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Bootstrap starting.
08:03:51.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.022291 21891 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Bootstrap starting.
08:03:51.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.023669 21890 tablet_bootstrap.cc:654] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Neither blocks nor log segments found. Creating new log.
08:03:51.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.026630 21890 log.cc:826] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Log is configured to *not* fsync() on all Append() calls
08:03:51.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.029351 21891 tablet_bootstrap.cc:654] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Neither blocks nor log segments found. Creating new log.
08:03:51.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.031381 21891 log.cc:826] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Log is configured to *not* fsync() on all Append() calls
08:03:51.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.043964 21891 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: No bootstrap required, opened a new log
08:03:51.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.044489 21891 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Time spent bootstrapping tablet: real 0.023s	user 0.019s	sys 0.000s
08:03:51.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.056432 21892 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap starting.
08:03:51.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.064580 21892 tablet_bootstrap.cc:654] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Neither blocks nor log segments found. Creating new log.
08:03:51.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.067467 21892 log.cc:826] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Log is configured to *not* fsync() on all Append() calls
08:03:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.077026 21891 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.077872 21891 raft_consensus.cc:385] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.078200 21891 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d31321473b59420090c29f4de9f6f928, State: Initialized, Role: FOLLOWER
08:03:51.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.079262 21891 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:51.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.082387 21890 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: No bootstrap required, opened a new log
08:03:51.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.082882 21890 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Time spent bootstrapping tablet: real 0.072s	user 0.023s	sys 0.026s
08:03:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.085765 21892 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: No bootstrap required, opened a new log
08:03:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.086560 21892 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Time spent bootstrapping tablet: real 0.031s	user 0.018s	sys 0.004s
08:03:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.089385 21891 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Time spent starting tablet: real 0.045s	user 0.028s	sys 0.012s
08:03:51.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.119567 21892 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:51.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.120334 21890 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:51.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.120815 21892 raft_consensus.cc:385] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:51.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.121218 21890 raft_consensus.cc:385] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:51.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.121511 21890 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e269425272f249e496b7c12b18f63076, State: Initialized, Role: FOLLOWER
08:03:51.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.121446 21892 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e8aea6cb22324b45b188c416f3d49634, State: Initialized, Role: FOLLOWER
08:03:51.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.122721 21890 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:51.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.123340 21892 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.126435 21890 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Time spent starting tablet: real 0.043s	user 0.031s	sys 0.000s
08:03:51.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.132483 21896 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:51.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.133199 21896 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:51.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.144279 21896 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:03:51.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.162487 21892 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Time spent starting tablet: real 0.075s	user 0.028s	sys 0.004s
08:03:51.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.163264 21859 tablet.cc:2404] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:51.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.178043 21528 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e8aea6cb22324b45b188c416f3d49634" is_pre_election: true
08:03:51.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.178889 21528 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d31321473b59420090c29f4de9f6f928 in term 0.
08:03:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.180387 21670 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076" is_pre_election: true
08:03:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.180439 21745 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e8aea6cb22324b45b188c416f3d49634; no voters: 
08:03:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.180964 21670 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d31321473b59420090c29f4de9f6f928 in term 0.
08:03:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.181298 21896 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:51.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.181649 21896 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:51.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.181984 21896 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 0 FOLLOWER]: Advancing to term 1
08:03:51.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.187155 21896 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:51.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.188519 21896 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 1 election: Requested vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:03:51.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.189301 21670 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076"
08:03:51.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.189451 21528 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e8aea6cb22324b45b188c416f3d49634"
08:03:51.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.189808 21670 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 0 FOLLOWER]: Advancing to term 1
08:03:51.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.189936 21528 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 0 FOLLOWER]: Advancing to term 1
08:03:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.196832 21670 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d31321473b59420090c29f4de9f6f928 in term 1.
08:03:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.196832 21528 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d31321473b59420090c29f4de9f6f928 in term 1.
08:03:51.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.197772 21745 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e8aea6cb22324b45b188c416f3d49634; no voters: 
08:03:51.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.198439 21896 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 1 FOLLOWER]: Leader election won for term 1
08:03:51.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.200297 21896 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 1 LEADER]: Becoming Leader. State: Replica: d31321473b59420090c29f4de9f6f928, State: Running, Role: LEADER
08:03:51.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.201702 21896 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:51.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.207934 21720 tablet.cc:2404] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:51.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.213439 21338 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 reported cstate change: term changed from 0 to 1, leader changed from <none> to d31321473b59420090c29f4de9f6f928 (127.20.164.67). New cstate: current_term: 1 leader_uuid: "d31321473b59420090c29f4de9f6f928" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: UNKNOWN } } }
08:03:51.260 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 5a6ef159b6014406b067e16f9a854c02
08:03:51.276 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:51.281 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299995, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:03:51.282 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [8ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=null) after lookup
08:03:51.296 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:03:51.298 [DEBUG - kudu-nio-3] (Connection.java:757) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] Successfully connected to peer
08:03:51.369 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x10863ab3, L:/127.0.0.1:36700 - R:/127.20.164.67:37251] using TOKEN/null
08:03:51.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.385632 21583 tablet.cc:2404] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.387053 21528 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 1 FOLLOWER]: Refusing update from remote peer d31321473b59420090c29f4de9f6f928: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.388631 21670 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 1 FOLLOWER]: Refusing update from remote peer d31321473b59420090c29f4de9f6f928: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.388717 21896 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:51.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.390404 21896 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.413638 21905 mvcc.cc:204] Tried to move back new op lower bound from 7287158502944428032 to 7287158502210097152. Current Snapshot: MvccSnapshot[applied={T|T < 7287158502944428032}]
08:03:51.559 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:51.561 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.164.67:37251
08:03:51.562 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:03:51.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.563406 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 21725
08:03:51.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:51.591 [DEBUG - kudu-nio-3] (Connection.java:239) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] handling channelInactive
08:03:51.591 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state READY due to: connection closed
08:03:51.592 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: connection closed
08:03:51.617 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:51.617 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:03:51.618 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [4ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@735587963(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@6409bcde -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@906641e -> wakeup thread Time-limited test)) after lookup
08:03:51.619 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:03:51.621 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.621 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.621 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.637 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:51.638 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:03:51.638 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=50), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [4ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [30ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [33ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [34ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [44ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [49ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@735587963(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@6409bcde -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@78713649 after retry RPC@723328259), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@906641e -> wakeup thread Time-limited test -> (continuation of Deferred@78713649 after retry RPC@723328259))) after lookup
08:03:51.639 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:03:51.642 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.642 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.642 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.657 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:51.657 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:03:51.658 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [4ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [30ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [33ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [34ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [44ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [49ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [51ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [54ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [54ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [68ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@735587963(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@6409bcde -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@78713649 after retry RPC@723328259) -> (continuation of Deferred@2027832046 after retry RPC@311582511), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@906641e -> wakeup thread Time-limited test -> (continuation of Deferred@78713649 after retry RPC@723328259) -> (continuation of Deferred@2027832046 after retry RPC@311582511))) after lookup
08:03:51.659 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:03:51.661 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.661 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.661 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.797 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:51.797 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:03:51.797 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=209), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [4ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [30ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [33ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [34ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [44ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [49ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [51ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [54ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [54ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [68ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [71ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [73ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [73ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [208ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@735587963(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@6409bcde -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@78713649 after retry RPC@723328259) -> (continuation of Deferred@2027832046 after retry RPC@311582511) -> (continuation of Deferred@824000686 after retry RPC@227707497), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@906641e -> wakeup thread Time-limited test -> (continuation of Deferred@78713649 after retry RPC@723328259) -> (continuation of Deferred@2027832046 after retry RPC@311582511) -> (continuation of Deferred@824000686 after retry RPC@227707497))) after lookup
08:03:51.798 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:03:51.799 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.800 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:51.800 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:52.117 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:52.117 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:03:52.117 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=529), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [4ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [30ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [33ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [34ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [44ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [49ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [51ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [54ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [54ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [68ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [71ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [73ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [73ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [208ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [210ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [212ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [212ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [525ms] refreshing cache from master, [525ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [528ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@735587963(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@6409bcde -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@78713649 after retry RPC@723328259) -> (continuation of Deferred@2027832046 after retry RPC@311582511) -> (continuation of Deferred@824000686 after retry RPC@227707497) -> (continuation of Deferred@1522184864 after retry RPC@1350197865), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@906641e -> wakeup thread Time-limited test -> (continuation of Deferred@78713649 after retry RPC@723328259) -> (continuation of Deferred@2027832046 after retry RPC@311582511) -> (continuation of Deferred@824000686 after retry RPC@227707497) -> (continuation of Deferred@1522184864 after retry RPC@1350197865))) after lookup
08:03:52.118 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:03:52.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:52.120 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:52.120 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:03:53.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.004076 21919 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d31321473b59420090c29f4de9f6f928)
08:03:53.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.004467 21920 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d31321473b59420090c29f4de9f6f928)
08:03:53.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.004954 21920 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:53.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.005328 21919 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.014082 21920 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:03:53.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.020372 21919 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:03:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.023399 21464 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:03:53.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.032070 21464 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:03:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.038408 21606 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:03:53.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.045706 21670 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e8aea6cb22324b45b188c416f3d49634" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076" is_pre_election: true
08:03:53.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.046285 21670 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e8aea6cb22324b45b188c416f3d49634 in term 1.
08:03:53.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.047714 21461 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e269425272f249e496b7c12b18f63076, e8aea6cb22324b45b188c416f3d49634; no voters: d31321473b59420090c29f4de9f6f928
08:03:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.048877 21919 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:03:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.049244 21919 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 1 FOLLOWER]: Starting leader election (detected failure of leader d31321473b59420090c29f4de9f6f928)
08:03:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.049579 21919 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 1 FOLLOWER]: Advancing to term 2
08:03:53.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.051473 21528 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e269425272f249e496b7c12b18f63076" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e8aea6cb22324b45b188c416f3d49634" is_pre_election: true
08:03:53.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.052064 21606 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:03:53.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.057081 21919 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.058374 21528 raft_consensus.cc:2393] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate e269425272f249e496b7c12b18f63076 in current term 2: Already voted for candidate e8aea6cb22324b45b188c416f3d49634 in this term.
08:03:53.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.063009 21605 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e269425272f249e496b7c12b18f63076; no voters: d31321473b59420090c29f4de9f6f928, e8aea6cb22324b45b188c416f3d49634
08:03:53.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.064042 21920 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 1 FOLLOWER]: Advancing to term 2
08:03:53.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.064103 21919 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 2 election: Requested vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:03:53.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.067225 21464 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:03:53.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.068416 21670 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e8aea6cb22324b45b188c416f3d49634" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076"
08:03:53.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.070631 21920 raft_consensus.cc:2749] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:03:53.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.076761 21670 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e8aea6cb22324b45b188c416f3d49634 in term 2.
08:03:53.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.079878 21461 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e269425272f249e496b7c12b18f63076, e8aea6cb22324b45b188c416f3d49634; no voters: d31321473b59420090c29f4de9f6f928
08:03:53.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.080595 21919 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 2 FOLLOWER]: Leader election won for term 2
08:03:53.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.082456 21919 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 2 LEADER]: Becoming Leader. State: Replica: e8aea6cb22324b45b188c416f3d49634, State: Running, Role: LEADER
08:03:53.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.083410 21919 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:53.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.091974 21338 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 reported cstate change: term changed from 1 to 2, leader changed from d31321473b59420090c29f4de9f6f928 (127.20.164.67) to e8aea6cb22324b45b188c416f3d49634 (127.20.164.65). New cstate: current_term: 2 leader_uuid: "e8aea6cb22324b45b188c416f3d49634" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: HEALTHY } } }
08:03:53.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.581224 21670 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 2 FOLLOWER]: Refusing update from remote peer e8aea6cb22324b45b188c416f3d49634: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
08:03:53.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.582345 21919 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
08:03:53.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.609615 21464 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 -> Peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Couldn't send request to peer d31321473b59420090c29f4de9f6f928. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:54.157 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:54.157 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:03:54.158 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2569), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [4ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [30ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [33ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [34ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [44ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [49ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [51ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [54ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [54ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [68ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [71ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [73ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [73ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [208ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [210ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [212ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [212ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [525ms] refreshing cache from master, [525ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [528ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [530ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [532ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [532ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2565ms] refreshing cache from master, [2565ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [2569ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@735587963(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@6409bcde -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@78713649 after retry RPC@723328259) -> (continuation of Deferred@2027832046 after retry RPC@311582511) -> (continuation of Deferred@824000686 after retry RPC@227707497) -> (continuation of Deferred@1522184864 after retry RPC@1350197865) -> (continuation of Deferred@495426522 after retry RPC@510516009), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@906641e -> wakeup thread Time-limited test -> (continuation of Deferred@78713649 after retry RPC@723328259) -> (continuation of Deferred@2027832046 after retry RPC@311582511) -> (continuation of Deferred@824000686 after retry RPC@227707497) -> (continuation of Deferred@1522184864 after retry RPC@1350197865) -> (continuation of Deferred@495426522 after retry RPC@510516009))) after lookup
08:03:54.158 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:03:54.159 [DEBUG - kudu-nio-1] (Connection.java:757) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] Successfully connected to peer
08:03:54.182 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x494dda5c, L:/127.0.0.1:42954 - R:/127.20.164.65:46489] using TOKEN/null
08:03:54.223 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.164.67:37251
08:03:54.225 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:03:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:54.226553 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/logs
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.67:37251
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.67
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42487
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.67
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:54.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:54.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:54.549190 21957 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:54.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:54.549645 21957 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:54.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:54.550066 21957 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:54.583848 21957 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:54.584681 21957 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.67
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:54.630100 21957 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.67:37251
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.67
08:03:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42487
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.67
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/logs
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:54.631372 21957 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:54.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:54.632912 21957 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:54.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:54.644927 21964 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:54.645665 21965 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:55.938699 21957 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.292s	user 0.007s	sys 0.001s
08:03:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:55.939230 21967 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:55.939465 21957 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.293s	user 0.009s	sys 0.001s
08:03:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.939956 21957 server_base.cc:1061] running on GCE node
08:03:55.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.942721 21957 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:55.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.945060 21957 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:55.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.946408 21957 hybrid_clock.cc:648] HybridClock initialized: now 1779091435946376 us; error 36 us; skew 500 ppm
08:03:55.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.947304 21957 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:55.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.954698 21957 webserver.cc:533] Webserver started at http://127.20.164.67:42487/ using document root <none> and password file <none>
08:03:55.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.955664 21957 fs_manager.cc:362] Metadata directory not provided
08:03:55.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.955953 21957 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:55.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.964308 21957 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.003s	sys 0.001s
08:03:55.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.969758 21979 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:55.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.970896 21957 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
08:03:55.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.971316 21957 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:03:55.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d31321473b59420090c29f4de9f6f928"
08:03:55.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:49 on dist-test-slave-0rvw"
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:55.974090 21957 fs_report.cc:389] FS layout report
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/data
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:56.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.069208 21957 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.071152 21957 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.071697 21957 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.075026 21957 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.081408 21986 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
08:03:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.089394 21957 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.089659 21957 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.000s
08:03:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.090008 21957 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:56.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.094434 21957 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:56.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.094636 21957 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
08:03:56.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.094973 21986 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Bootstrap starting.
08:03:56.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:56.132738 21464 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 -> Peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Couldn't send request to peer d31321473b59420090c29f4de9f6f928. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:03:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.147414 21986 log.cc:826] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Log is configured to *not* fsync() on all Append() calls
08:03:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.176834 21986 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.177676 21986 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Bootstrap complete.
08:03:56.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.179502 21986 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Time spent bootstrapping tablet: real 0.085s	user 0.042s	sys 0.041s
08:03:56.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.198398 21986 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:56.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.201265 21986 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d31321473b59420090c29f4de9f6f928, State: Initialized, Role: FOLLOWER
08:03:56.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.202630 21986 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:56.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.207727 21986 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Time spent starting tablet: real 0.028s	user 0.021s	sys 0.007s
08:03:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.262179 21957 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.67:37251
08:03:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.262538 22096 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.67:37251 every 8 connection(s)
08:03:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.264864 21957 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:03:56.273 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:56.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.272800 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 21957
08:03:56.302 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:56.303 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.164.65:46489
08:03:56.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:03:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.304150 22098 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:03:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.304678 22098 heartbeater.cc:461] Registering TS with master...
08:03:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.305298 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 21422
08:03:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.306069 22098 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:03:56.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.310539 22099 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:03:56.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.310873 22099 heartbeater.cc:461] Registering TS with master...
08:03:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.311753 22099 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:03:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.312382 22097 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:03:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.312754 22097 heartbeater.cc:461] Registering TS with master...
08:03:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.313624 22097 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:03:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.316268 21183 ts_manager.cc:194] Re-registered known tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:03:56.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.316860 21337 ts_manager.cc:194] Re-registered known tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:03:56.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.317633 21258 ts_manager.cc:194] Re-registered known tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:03:56.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:56.320114 21337 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.67:41521
08:03:56.339 [DEBUG - kudu-nio-1] (Connection.java:239) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] handling channelInactive
08:03:56.340 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:56.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state READY due to: connection closed
08:03:56.341 [DEBUG - Time-limited test] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:03:56.342 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.343 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.343 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.355 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:56.356 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:03:56.356 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1423322900(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@49fe27f4 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1bf91c4c -> wakeup thread Time-limited test)) after lookup
08:03:56.357 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:03:56.358 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.358 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.358 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.376 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:56.376 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:03:56.377 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [15ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1423322900(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@49fe27f4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1bf91c4c -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633))) after lookup
08:03:56.377 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:03:56.378 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.378 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.378 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.396 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:56.396 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:03:56.396 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=55), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [15ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [36ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [37ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [37ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1423322900(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@49fe27f4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633) -> (continuation of Deferred@1407071974 after retry RPC@116932205), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1bf91c4c -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633) -> (continuation of Deferred@1407071974 after retry RPC@116932205))) after lookup
08:03:56.397 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:03:56.398 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.398 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.398 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.436 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:56.437 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:03:56.437 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=96), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [15ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [36ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [37ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [37ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [56ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [57ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [57ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [92ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [95ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1423322900(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@49fe27f4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633) -> (continuation of Deferred@1407071974 after retry RPC@116932205) -> (continuation of Deferred@702787031 after retry RPC@1857025488), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1bf91c4c -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633) -> (continuation of Deferred@1407071974 after retry RPC@116932205) -> (continuation of Deferred@702787031 after retry RPC@1857025488))) after lookup
08:03:56.438 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:03:56.438 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.439 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.439 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.596 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:56.597 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:03:56.597 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=256), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [15ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [36ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [37ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [37ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [56ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [57ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [57ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [92ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [95ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [96ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [98ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [98ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [251ms] refreshing cache from master, [251ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [255ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1423322900(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@49fe27f4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633) -> (continuation of Deferred@1407071974 after retry RPC@116932205) -> (continuation of Deferred@702787031 after retry RPC@1857025488) -> (continuation of Deferred@1632911184 after retry RPC@1883527135), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1bf91c4c -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633) -> (continuation of Deferred@1407071974 after retry RPC@116932205) -> (continuation of Deferred@702787031 after retry RPC@1857025488) -> (continuation of Deferred@1632911184 after retry RPC@1883527135))) after lookup
08:03:56.598 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:03:56.599 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.599 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.599 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.836 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:56.837 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:03:56.837 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=496), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [15ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [36ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [37ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [37ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [56ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [57ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [57ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [92ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [95ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [96ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [98ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [98ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [251ms] refreshing cache from master, [251ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [255ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [256ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [258ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [258ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [491ms] refreshing cache from master, [492ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [495ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1423322900(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@49fe27f4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633) -> (continuation of Deferred@1407071974 after retry RPC@116932205) -> (continuation of Deferred@702787031 after retry RPC@1857025488) -> (continuation of Deferred@1632911184 after retry RPC@1883527135) -> (continuation of Deferred@48678670 after retry RPC@172778175), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1bf91c4c -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633) -> (continuation of Deferred@1407071974 after retry RPC@116932205) -> (continuation of Deferred@702787031 after retry RPC@1857025488) -> (continuation of Deferred@1632911184 after retry RPC@1883527135) -> (continuation of Deferred@48678670 after retry RPC@172778175))) after lookup
08:03:56.838 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:03:56.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:56.839 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:03:57.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.323004 22097 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:03:57.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.422557 22110 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:57.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.423049 22110 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.426540 22110 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:03:57.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:57.435456 21982 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:03:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:57.440069 21982 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:03:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.442668 21670 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076" is_pre_election: true
08:03:57.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.444279 21980 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d31321473b59420090c29f4de9f6f928; no voters: e269425272f249e496b7c12b18f63076, e8aea6cb22324b45b188c416f3d49634
08:03:57.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.444985 22110 raft_consensus.cc:2749] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:03:57.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.474565 22115 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader e8aea6cb22324b45b188c416f3d49634)
08:03:57.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.475005 22115 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:57.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.476866 22115 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:03:57.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:57.484231 21605 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:03:57.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.488085 22052 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e269425272f249e496b7c12b18f63076" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "d31321473b59420090c29f4de9f6f928" is_pre_election: true
08:03:57.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.488673 22052 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e269425272f249e496b7c12b18f63076 in term 1.
08:03:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.489768 21606 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e269425272f249e496b7c12b18f63076; no voters: e8aea6cb22324b45b188c416f3d49634
08:03:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.490353 22115 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 2 FOLLOWER]: Leader pre-election won for term 3
08:03:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.490640 22115 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 2 FOLLOWER]: Starting leader election (detected failure of leader e8aea6cb22324b45b188c416f3d49634)
08:03:57.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.490849 22115 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 2 FOLLOWER]: Advancing to term 3
08:03:57.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.495254 22115 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:57.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.496608 22115 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 3 election: Requested vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:03:57.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.497298 22052 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e269425272f249e496b7c12b18f63076" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "d31321473b59420090c29f4de9f6f928"
08:03:57.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.497819 22052 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 1 FOLLOWER]: Advancing to term 3
08:03:57.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:57.500092 21605 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:03:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.504546 22052 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e269425272f249e496b7c12b18f63076 in term 3.
08:03:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.505254 21606 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e269425272f249e496b7c12b18f63076; no voters: e8aea6cb22324b45b188c416f3d49634
08:03:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.505786 22115 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 3 FOLLOWER]: Leader election won for term 3
08:03:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.507285 22115 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 3 LEADER]: Becoming Leader. State: Replica: e269425272f249e496b7c12b18f63076, State: Running, Role: LEADER
08:03:57.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.508329 22115 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:03:57.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.518416 21337 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 reported cstate change: term changed from 2 to 3, leader changed from e8aea6cb22324b45b188c416f3d49634 (127.20.164.65) to e269425272f249e496b7c12b18f63076 (127.20.164.66). New cstate: current_term: 3 leader_uuid: "e269425272f249e496b7c12b18f63076" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: UNKNOWN } } }
08:03:57.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.903424 22052 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 3 FOLLOWER]: Refusing update from remote peer e269425272f249e496b7c12b18f63076: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
08:03:57.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:57.904685 22115 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
08:03:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:57.934057 21605 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 -> Peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Couldn't send request to peer e8aea6cb22324b45b188c416f3d49634. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:58.426079 21605 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111) [suppressed 3 similar messages]
08:03:59.136 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:03:59.136 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:03:59.137 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=2795), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [15ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [36ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [37ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [37ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [56ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [57ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [57ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [92ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [95ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [96ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [98ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [98ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [251ms] refreshing cache from master, [251ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [255ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [256ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [258ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [258ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [491ms] refreshing cache from master, [492ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [495ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [496ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [498ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [498ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2791ms] refreshing cache from master, [2791ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [2795ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1423322900(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@49fe27f4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633) -> (continuation of Deferred@1407071974 after retry RPC@116932205) -> (continuation of Deferred@702787031 after retry RPC@1857025488) -> (continuation of Deferred@1632911184 after retry RPC@1883527135) -> (continuation of Deferred@48678670 after retry RPC@172778175) -> (continuation of Deferred@783499073 after retry RPC@1191442866), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1bf91c4c -> wakeup thread Time-limited test -> (continuation of Deferred@2037879183 after retry RPC@708831633) -> (continuation of Deferred@1407071974 after retry RPC@116932205) -> (continuation of Deferred@702787031 after retry RPC@1857025488) -> (continuation of Deferred@1632911184 after retry RPC@1883527135) -> (continuation of Deferred@48678670 after retry RPC@172778175) -> (continuation of Deferred@783499073 after retry RPC@1191442866))) after lookup
08:03:59.138 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:03:59.139 [DEBUG - kudu-nio-1] (Connection.java:757) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] Successfully connected to peer
08:03:59.159 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7613a087, L:/127.0.0.1:37116 - R:/127.20.164.66:39351] using TOKEN/null
08:03:59.210 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.164.65:46489
08:03:59.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:59.213176 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/logs
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.65:46489
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.65
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33865
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.65
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:59.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:59.515355 22150 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:59.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:59.515933 22150 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:59.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:59.516531 22150 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:59.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:59.551864 22150 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:59.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:59.552803 22150 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.65
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:59.588825 22150 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.65:46489
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.65
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33865
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.65
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/logs
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:03:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:03:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:59.590502 22150 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:59.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:59.592278 22150 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:59.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:59.604226 22158 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:00.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:00.265653 21605 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 -> Peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Couldn't send request to peer e8aea6cb22324b45b188c416f3d49634. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:04:00.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:59.605316 22159 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:00.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:00.856721 22161 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:00.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:00.857451 22160 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1249 milliseconds
08:04:00.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.857473 22150 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.861915 22150 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:00.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.864238 22150 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:00.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.865564 22150 hybrid_clock.cc:648] HybridClock initialized: now 1779091440865549 us; error 48 us; skew 500 ppm
08:04:00.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.866266 22150 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:00.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.873507 22150 webserver.cc:533] Webserver started at http://127.20.164.65:33865/ using document root <none> and password file <none>
08:04:00.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.874375 22150 fs_manager.cc:362] Metadata directory not provided
08:04:00.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.874562 22150 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.881899 22150 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.000s	sys 0.006s
08:04:00.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.886488 22170 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:04:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.887820 22150 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.888178 22150 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e8aea6cb22324b45b188c416f3d49634"
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:44 on dist-test-slave-0rvw"
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.890038 22150 fs_report.cc:389] FS layout report
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/data
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:00.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.954725 22150 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:04:00.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.956748 22150 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:00.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.957307 22150 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:00.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.960860 22150 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:00.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.966349 22177 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
08:04:00.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.974851 22150 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:04:00.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.975121 22150 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
08:04:00.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.975334 22150 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:04:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.979972 22150 ts_tablet_manager.cc:616] Registered 1 tablets
08:04:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.980147 22150 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
08:04:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:00.980680 22177 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap starting.
08:04:01.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.043484 22177 log.cc:826] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Log is configured to *not* fsync() on all Append() calls
08:04:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.105682 22177 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.106881 22177 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap complete.
08:04:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.108928 22177 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Time spent bootstrapping tablet: real 0.129s	user 0.086s	sys 0.037s
08:04:01.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.139041 22177 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:01.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.142802 22177 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: e8aea6cb22324b45b188c416f3d49634, State: Initialized, Role: FOLLOWER
08:04:01.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.143896 22177 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.148952 22177 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Time spent starting tablet: real 0.040s	user 0.023s	sys 0.016s
08:04:01.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.174944 22150 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.65:46489
08:04:01.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.175246 22285 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.65:46489 every 8 connection(s)
08:04:01.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.177369 22150 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:04:01.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.185465 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 22150
08:04:01.186 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:01.220 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:04:01.224 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.164.66:39351
08:04:01.224 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:04:01.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.225366 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 21575
08:04:01.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.232134 22288 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:04:01.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.232635 22288 heartbeater.cc:461] Registering TS with master...
08:04:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.233982 22288 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:04:01.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.238005 22286 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:04:01.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.238370 22286 heartbeater.cc:461] Registering TS with master...
08:04:01.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.238464 22287 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:04:01.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.238826 22287 heartbeater.cc:461] Registering TS with master...
08:04:01.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.238996 21182 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:01.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.239403 22286 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:04:01.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.239749 22287 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:04:01.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.243435 21338 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:01.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.243593 21258 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:01.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:01.246363 21338 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.65:54501
08:04:01.261 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:01.261 [DEBUG - kudu-nio-1] (Connection.java:239) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] handling channelInactive
08:04:01.261 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state READY due to: connection closed
08:04:01.273 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:01.275 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.275 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.276 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.297 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:01.297 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:01.298 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=3, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [1ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server e269425272f249e496b7c12b18f63076, [15ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [15ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@866192102(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@e5370d9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5efe8375 -> wakeup thread Time-limited test)) after lookup
08:04:01.298 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:01.299 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.299 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.299 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.316 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:01.316 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:01.316 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=5, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=55), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [1ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server e269425272f249e496b7c12b18f63076, [15ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [15ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [37ms] sending RPC to server e269425272f249e496b7c12b18f63076, [38ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [38ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@866192102(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@e5370d9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@64068433 after retry RPC@698242421), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5efe8375 -> wakeup thread Time-limited test -> (continuation of Deferred@64068433 after retry RPC@698242421))) after lookup
08:04:01.317 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:01.318 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.318 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.318 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.357 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:01.357 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:01.358 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=7, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=96), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [1ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server e269425272f249e496b7c12b18f63076, [15ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [15ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [37ms] sending RPC to server e269425272f249e496b7c12b18f63076, [38ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [38ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [56ms] sending RPC to server e269425272f249e496b7c12b18f63076, [57ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [57ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [92ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [96ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@866192102(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@e5370d9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@64068433 after retry RPC@698242421) -> (continuation of Deferred@1942568119 after retry RPC@682097222), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5efe8375 -> wakeup thread Time-limited test -> (continuation of Deferred@64068433 after retry RPC@698242421) -> (continuation of Deferred@1942568119 after retry RPC@682097222))) after lookup
08:04:01.358 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:01.359 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.359 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.359 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.456 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:01.456 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:01.456 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=9, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=195), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [1ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server e269425272f249e496b7c12b18f63076, [15ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [15ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [37ms] sending RPC to server e269425272f249e496b7c12b18f63076, [38ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [38ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [56ms] sending RPC to server e269425272f249e496b7c12b18f63076, [57ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [57ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [92ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [96ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [97ms] sending RPC to server e269425272f249e496b7c12b18f63076, [98ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [98ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [194ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@866192102(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@e5370d9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@64068433 after retry RPC@698242421) -> (continuation of Deferred@1942568119 after retry RPC@682097222) -> (continuation of Deferred@1971885557 after retry RPC@1659516104), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5efe8375 -> wakeup thread Time-limited test -> (continuation of Deferred@64068433 after retry RPC@698242421) -> (continuation of Deferred@1942568119 after retry RPC@682097222) -> (continuation of Deferred@1971885557 after retry RPC@1659516104))) after lookup
08:04:01.457 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:01.458 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.458 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.458 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.976 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:01.976 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:01.976 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=11, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=715), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [1ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server e269425272f249e496b7c12b18f63076, [15ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [15ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [37ms] sending RPC to server e269425272f249e496b7c12b18f63076, [38ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [38ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [56ms] sending RPC to server e269425272f249e496b7c12b18f63076, [57ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [57ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [92ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [96ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [97ms] sending RPC to server e269425272f249e496b7c12b18f63076, [98ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [98ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [194ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [196ms] sending RPC to server e269425272f249e496b7c12b18f63076, [197ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [197ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [711ms] refreshing cache from master, [711ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [714ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@866192102(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@e5370d9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@64068433 after retry RPC@698242421) -> (continuation of Deferred@1942568119 after retry RPC@682097222) -> (continuation of Deferred@1971885557 after retry RPC@1659516104) -> (continuation of Deferred@1556604831 after retry RPC@1865991355), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5efe8375 -> wakeup thread Time-limited test -> (continuation of Deferred@64068433 after retry RPC@698242421) -> (continuation of Deferred@1942568119 after retry RPC@682097222) -> (continuation of Deferred@1971885557 after retry RPC@1659516104) -> (continuation of Deferred@1556604831 after retry RPC@1865991355))) after lookup
08:04:01.977 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:01.978 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.978 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:01.978 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.249265 22286 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.646427 22297 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader e269425272f249e496b7c12b18f63076)
08:04:02.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.646809 22297 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:02.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.648169 22297 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:02.650650 21980 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111)
08:04:02.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:02.653859 21980 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111)
08:04:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.659673 22240 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "e8aea6cb22324b45b188c416f3d49634" is_pre_election: true
08:04:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.660261 22240 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d31321473b59420090c29f4de9f6f928 in term 2.
08:04:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.661290 21982 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e8aea6cb22324b45b188c416f3d49634; no voters: e269425272f249e496b7c12b18f63076
08:04:02.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.661827 22297 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 3 FOLLOWER]: Leader pre-election won for term 4
08:04:02.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.662103 22297 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 3 FOLLOWER]: Starting leader election (detected failure of leader e269425272f249e496b7c12b18f63076)
08:04:02.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.662428 22297 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 3 FOLLOWER]: Advancing to term 4
08:04:02.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.666879 22297 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:02.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.668360 22297 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 4 election: Requested vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:02.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.669600 22240 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "e8aea6cb22324b45b188c416f3d49634"
08:04:02.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.670113 22240 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 2 FOLLOWER]: Advancing to term 4
08:04:02.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:02.672290 21980 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111)
08:04:02.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.676803 22240 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d31321473b59420090c29f4de9f6f928 in term 4.
08:04:02.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.677515 21982 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e8aea6cb22324b45b188c416f3d49634; no voters: e269425272f249e496b7c12b18f63076
08:04:02.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.678059 22297 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 4 FOLLOWER]: Leader election won for term 4
08:04:02.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.679145 22297 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 4 LEADER]: Becoming Leader. State: Replica: d31321473b59420090c29f4de9f6f928, State: Running, Role: LEADER
08:04:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.679895 22297 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:02.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:02.687008 21338 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 reported cstate change: term changed from 3 to 4, leader changed from e269425272f249e496b7c12b18f63076 (127.20.164.66) to d31321473b59420090c29f4de9f6f928 (127.20.164.67). New cstate: current_term: 4 leader_uuid: "d31321473b59420090c29f4de9f6f928" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: UNKNOWN } } }
08:04:03.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:03.124812 21980 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 -> Peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Couldn't send request to peer e269425272f249e496b7c12b18f63076. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:03.281365 22240 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 4 FOLLOWER]: Refusing update from remote peer d31321473b59420090c29f4de9f6f928: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
08:04:03.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:03.282794 22297 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
08:04:04.797 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:04.798 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:04.798 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=13, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3536), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [1ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server e269425272f249e496b7c12b18f63076, [15ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [15ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [37ms] sending RPC to server e269425272f249e496b7c12b18f63076, [38ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [38ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [56ms] sending RPC to server e269425272f249e496b7c12b18f63076, [57ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [57ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [92ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [96ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [97ms] sending RPC to server e269425272f249e496b7c12b18f63076, [98ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [98ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [194ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [196ms] sending RPC to server e269425272f249e496b7c12b18f63076, [197ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [197ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [711ms] refreshing cache from master, [711ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [714ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [716ms] sending RPC to server e269425272f249e496b7c12b18f63076, [717ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [717ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [3532ms] refreshing cache from master, [3532ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [3536ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@866192102(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@e5370d9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@64068433 after retry RPC@698242421) -> (continuation of Deferred@1942568119 after retry RPC@682097222) -> (continuation of Deferred@1971885557 after retry RPC@1659516104) -> (continuation of Deferred@1556604831 after retry RPC@1865991355) -> (continuation of Deferred@995647352 after retry RPC@317475252), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5efe8375 -> wakeup thread Time-limited test -> (continuation of Deferred@64068433 after retry RPC@698242421) -> (continuation of Deferred@1942568119 after retry RPC@682097222) -> (continuation of Deferred@1971885557 after retry RPC@1659516104) -> (continuation of Deferred@1556604831 after retry RPC@1865991355) -> (continuation of Deferred@995647352 after retry RPC@317475252))) after lookup
08:04:04.799 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:04.800 [DEBUG - kudu-nio-0] (Connection.java:757) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] Successfully connected to peer
08:04:04.832 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x5164a965, L:/127.0.0.1:44878 - R:/127.20.164.67:37251] using TOKEN/null
08:04:04.882 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.164.66:39351
08:04:04.883 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:04:04.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:04.884538 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/logs
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.66:39351
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.66
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35829
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.66
08:04:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:04:05.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:05.199533 22338 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:05.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:05.200038 22338 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:05.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:05.200466 22338 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:05.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:05.237905 22338 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:05.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:05.238675 22338 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.66
08:04:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:05.282611 22338 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data
08:04:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.66:39351
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.66
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35829
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.66
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/logs
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:04:05.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:04:05.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:05.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:05.284093 22338 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:05.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:05.286132 22338 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:05.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:05.301394 22347 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:05.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:05.853323 21980 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 -> Peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Couldn't send request to peer e269425272f249e496b7c12b18f63076. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:04:06.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:06.703732 22346 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 22338
08:04:06.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:05.303349 22348 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:06.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:06.986904 22338 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.686s	user 0.573s	sys 1.044s
08:04:06.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:06.987299 22338 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.687s	user 0.573s	sys 1.044s
08:04:06.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:06.989424 22350 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:06.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:06.991376 22349 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1686 milliseconds
08:04:06.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:06.991437 22338 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:06.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:06.992645 22338 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:06.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:06.994686 22338 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:06.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:06.996044 22338 hybrid_clock.cc:648] HybridClock initialized: now 1779091446996008 us; error 33 us; skew 500 ppm
08:04:06.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:06.996836 22338 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:07.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.003185 22338 webserver.cc:533] Webserver started at http://127.20.164.66:35829/ using document root <none> and password file <none>
08:04:07.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.004304 22338 fs_manager.cc:362] Metadata directory not provided
08:04:07.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.004561 22338 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:07.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.014317 22338 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.005s	sys 0.000s
08:04:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.019665 22363 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:04:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.020915 22338 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
08:04:07.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.021282 22338 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:07.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e269425272f249e496b7c12b18f63076"
08:04:07.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:47 on dist-test-slave-0rvw"
08:04:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.023846 22338 fs_report.cc:389] FS layout report
08:04:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/data
08:04:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:07.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:07.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:07.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:07.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:07.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:07.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:07.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:07.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:07.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:07.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.103058 22338 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:04:07.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.104682 22338 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:07.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.105057 22338 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:07.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.107501 22338 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:07.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.113271 22370 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
08:04:07.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.121156 22338 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:04:07.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.121395 22338 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.000s	sys 0.002s
08:04:07.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.121613 22338 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:04:07.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.126053 22338 ts_tablet_manager.cc:616] Registered 1 tablets
08:04:07.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.126302 22338 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
08:04:07.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.126718 22370 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Bootstrap starting.
08:04:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.185361 22370 log.cc:826] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Log is configured to *not* fsync() on all Append() calls
08:04:07.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.241665 22370 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.242810 22370 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Bootstrap complete.
08:04:07.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.244282 22370 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Time spent bootstrapping tablet: real 0.118s	user 0.079s	sys 0.036s
08:04:07.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.265123 22370 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.268595 22370 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: e269425272f249e496b7c12b18f63076, State: Initialized, Role: FOLLOWER
08:04:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.269675 22370 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:07.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.273512 22370 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Time spent starting tablet: real 0.029s	user 0.025s	sys 0.004s
08:04:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.310797 22338 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.66:39351
08:04:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.311156 22478 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.66:39351 every 8 connection(s)
08:04:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.313866 22338 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:04:07.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.324620 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 22338
08:04:07.325 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:07.372 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:04:07.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.377065 22480 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:04:07.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.377578 22480 heartbeater.cc:461] Registering TS with master...
08:04:07.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.378815 22480 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:04:07.385 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.164.67:37251
08:04:07.385 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:04:07.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.384109 21257 ts_manager.cc:194] Re-registered known tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:07.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.385708 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 21957
08:04:07.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.397974 22479 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:04:07.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.398105 22481 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:04:07.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.398373 22479 heartbeater.cc:461] Registering TS with master...
08:04:07.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.398454 22481 heartbeater.cc:461] Registering TS with master...
08:04:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.399390 22479 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:04:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.399451 22481 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:04:07.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.402590 21338 ts_manager.cc:194] Re-registered known tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:07.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.403288 21182 ts_manager.cc:194] Re-registered known tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:07.405260 21338 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.66:40261
08:04:07.421 [DEBUG - kudu-nio-0] (Connection.java:239) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] handling channelInactive
08:04:07.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:07.421 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state READY due to: connection closed
08:04:07.421 [DEBUG - Time-limited test] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:07.422 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.423 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.423 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.436 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:07.437 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:07.437 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@182744335(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@55ff520b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22275b9f -> wakeup thread Time-limited test)) after lookup
08:04:07.437 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:07.438 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.439 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.439 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.455 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:07.455 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:07.455 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [18ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [18ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@182744335(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@55ff520b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@698149982 after retry RPC@838840657), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22275b9f -> wakeup thread Time-limited test -> (continuation of Deferred@698149982 after retry RPC@838840657))) after lookup
08:04:07.456 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:07.457 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.457 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.458 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.475 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:07.475 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:07.475 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=54), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [18ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [18ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [35ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [36ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [37ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@182744335(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@55ff520b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@698149982 after retry RPC@838840657) -> (continuation of Deferred@2089744251 after retry RPC@817345666), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22275b9f -> wakeup thread Time-limited test -> (continuation of Deferred@698149982 after retry RPC@838840657) -> (continuation of Deferred@2089744251 after retry RPC@817345666))) after lookup
08:04:07.476 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:07.477 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.477 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.477 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.596 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:07.597 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:07.597 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=175), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [18ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [18ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [35ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [36ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [37ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [55ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [56ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [56ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [175ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@182744335(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@55ff520b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@698149982 after retry RPC@838840657) -> (continuation of Deferred@2089744251 after retry RPC@817345666) -> (continuation of Deferred@1209129019 after retry RPC@1167025695), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22275b9f -> wakeup thread Time-limited test -> (continuation of Deferred@698149982 after retry RPC@838840657) -> (continuation of Deferred@2089744251 after retry RPC@817345666) -> (continuation of Deferred@1209129019 after retry RPC@1167025695))) after lookup
08:04:07.598 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:07.599 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.599 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.600 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.836 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:07.837 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:07.837 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=416), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [18ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [18ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [35ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [36ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [37ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [55ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [56ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [56ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [175ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [177ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [178ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [179ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [411ms] refreshing cache from master, [411ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [415ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@182744335(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@55ff520b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@698149982 after retry RPC@838840657) -> (continuation of Deferred@2089744251 after retry RPC@817345666) -> (continuation of Deferred@1209129019 after retry RPC@1167025695) -> (continuation of Deferred@2142439919 after retry RPC@1102194584), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22275b9f -> wakeup thread Time-limited test -> (continuation of Deferred@698149982 after retry RPC@838840657) -> (continuation of Deferred@2089744251 after retry RPC@817345666) -> (continuation of Deferred@1209129019 after retry RPC@1167025695) -> (continuation of Deferred@2142439919 after retry RPC@1102194584))) after lookup
08:04:07.838 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:07.839 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.839 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:07.840 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:08.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.408694 22479 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:04:08.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.570609 22491 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d31321473b59420090c29f4de9f6f928)
08:04:08.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.571056 22491 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.573379 22491 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:08.575858 22174 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:04:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:08.579727 22174 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:04:08.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.585229 22433 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e8aea6cb22324b45b188c416f3d49634" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076" is_pre_election: true
08:04:08.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.585927 22433 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e8aea6cb22324b45b188c416f3d49634 in term 3.
08:04:08.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.587314 22171 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e269425272f249e496b7c12b18f63076, e8aea6cb22324b45b188c416f3d49634; no voters: d31321473b59420090c29f4de9f6f928
08:04:08.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.587940 22491 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 4 FOLLOWER]: Leader pre-election won for term 5
08:04:08.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.588204 22491 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 4 FOLLOWER]: Starting leader election (detected failure of leader d31321473b59420090c29f4de9f6f928)
08:04:08.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.588567 22491 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 4 FOLLOWER]: Advancing to term 5
08:04:08.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.592845 22491 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:08.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.594498 22491 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 5 election: Requested vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:08.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.595425 22433 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e8aea6cb22324b45b188c416f3d49634" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076"
08:04:08.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.595841 22433 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 3 FOLLOWER]: Advancing to term 5
08:04:08.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:08.597417 22174 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:04:08.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.601828 22433 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e8aea6cb22324b45b188c416f3d49634 in term 5.
08:04:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.602746 22171 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e269425272f249e496b7c12b18f63076, e8aea6cb22324b45b188c416f3d49634; no voters: d31321473b59420090c29f4de9f6f928
08:04:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.603307 22491 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 5 FOLLOWER]: Leader election won for term 5
08:04:08.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.604736 22491 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 5 LEADER]: Becoming Leader. State: Replica: e8aea6cb22324b45b188c416f3d49634, State: Running, Role: LEADER
08:04:08.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.605559 22491 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:08.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:08.613652 21338 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 reported cstate change: term changed from 4 to 5, leader changed from d31321473b59420090c29f4de9f6f928 (127.20.164.67) to e8aea6cb22324b45b188c416f3d49634 (127.20.164.65). New cstate: current_term: 5 leader_uuid: "e8aea6cb22324b45b188c416f3d49634" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: HEALTHY } } }
08:04:09.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:09.130090 22433 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 5 FOLLOWER]: Refusing update from remote peer e8aea6cb22324b45b188c416f3d49634: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
08:04:09.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:09.131644 22508 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.001s
08:04:09.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:09.138095 22174 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 -> Peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Couldn't send request to peer d31321473b59420090c29f4de9f6f928. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:09.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:09.145685 22513 mvcc.cc:204] Tried to move back new op lower bound from 7287158558071267328 to 7287158549224775680. Current Snapshot: MvccSnapshot[applied={T|T < 7287158558071267328}]
08:04:09.317 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:09.317 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:09.318 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1896), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [11ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [15ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [18ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [18ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [35ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [36ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [37ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [55ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [56ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [56ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [175ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [177ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [178ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [179ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [411ms] refreshing cache from master, [411ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [415ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [417ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [418ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [419ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [1892ms] refreshing cache from master, [1892ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [1895ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@182744335(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@55ff520b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@698149982 after retry RPC@838840657) -> (continuation of Deferred@2089744251 after retry RPC@817345666) -> (continuation of Deferred@1209129019 after retry RPC@1167025695) -> (continuation of Deferred@2142439919 after retry RPC@1102194584) -> (continuation of Deferred@208265025 after retry RPC@1555543987), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22275b9f -> wakeup thread Time-limited test -> (continuation of Deferred@698149982 after retry RPC@838840657) -> (continuation of Deferred@2089744251 after retry RPC@817345666) -> (continuation of Deferred@1209129019 after retry RPC@1167025695) -> (continuation of Deferred@2142439919 after retry RPC@1102194584) -> (continuation of Deferred@208265025 after retry RPC@1555543987))) after lookup
08:04:09.319 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:09.320 [DEBUG - kudu-nio-7] (Connection.java:757) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] Successfully connected to peer
08:04:09.348 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0xa680d4ed, L:/127.0.0.1:45824 - R:/127.20.164.65:46489] using TOKEN/null
08:04:09.386 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.164.67:37251
08:04:09.387 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:04:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:09.388147 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:04:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data
08:04:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/logs
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.67:37251
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.67
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42487
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.67
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:04:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:09.695084 22522 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:09.695634 22522 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:09.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:09.696215 22522 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:09.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:09.730933 22522 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:09.731878 22522 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.67
08:04:09.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:09.767526 22522 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data
08:04:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:04:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.67:37251
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.67
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42487
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.67
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/logs
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:09.769202 22522 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:09.771067 22522 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:09.783576 22530 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:09.788278 22533 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:09.784492 22531 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:10.960424 22532 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:04:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.960453 22522 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.964632 22522 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.966639 22522 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.967990 22522 hybrid_clock.cc:648] HybridClock initialized: now 1779091450967964 us; error 56 us; skew 500 ppm
08:04:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.968729 22522 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.974798 22522 webserver.cc:533] Webserver started at http://127.20.164.67:42487/ using document root <none> and password file <none>
08:04:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.975672 22522 fs_manager.cc:362] Metadata directory not provided
08:04:10.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.975847 22522 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.983192 22522 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.002s
08:04:10.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.987604 22540 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:04:10.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.988706 22522 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
08:04:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.988984 22522 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:04:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d31321473b59420090c29f4de9f6f928"
08:04:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:49 on dist-test-slave-0rvw"
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:10.990772 22522 fs_report.cc:389] FS layout report
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/data
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:11.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.050258 22522 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:04:11.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.051716 22522 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:11.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.052103 22522 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.054673 22522 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:11.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.060514 22547 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
08:04:11.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.068401 22522 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:04:11.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.068602 22522 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
08:04:11.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.068802 22522 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:04:11.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.073293 22522 ts_tablet_manager.cc:616] Registered 1 tablets
08:04:11.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.073465 22522 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
08:04:11.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.073915 22547 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Bootstrap starting.
08:04:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.132194 22547 log.cc:826] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Log is configured to *not* fsync() on all Append() calls
08:04:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.189186 22547 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.190219 22547 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Bootstrap complete.
08:04:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.191876 22547 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Time spent bootstrapping tablet: real 0.118s	user 0.085s	sys 0.028s
08:04:11.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.209141 22547 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:11.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.211553 22547 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: d31321473b59420090c29f4de9f6f928, State: Initialized, Role: FOLLOWER
08:04:11.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.212551 22547 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.216948 22547 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Time spent starting tablet: real 0.025s	user 0.020s	sys 0.003s
08:04:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.264294 22522 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.67:37251
08:04:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.264624 22655 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.67:37251 every 8 connection(s)
08:04:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.266644 22522 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:04:11.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.271515 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 22522
08:04:11.272 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:11.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.300952 22656 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:04:11.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.301417 22656 heartbeater.cc:461] Registering TS with master...
08:04:11.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.302683 22656 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:04:11.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.308010 21338 ts_manager.cc:194] Re-registered known tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:04:11.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.311691 22658 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:04:11.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.312050 22658 heartbeater.cc:461] Registering TS with master...
08:04:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.312987 22658 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:04:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.313966 22657 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:04:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.314313 22657 heartbeater.cc:461] Registering TS with master...
08:04:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.315222 22657 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:04:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.316437 21182 ts_manager.cc:194] Re-registered known tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:04:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.317654 21338 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.67:38029
08:04:11.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.319182 21257 ts_manager.cc:194] Re-registered known tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:04:11.324 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:04:11.324 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.164.65:46489
08:04:11.325 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:04:11.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:11.325415 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 22150
08:04:11.351 [DEBUG - kudu-nio-7] (Connection.java:239) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] handling channelInactive
08:04:11.351 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:11.352 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state READY due to: connection closed
08:04:11.352 [DEBUG - Time-limited test] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:11.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.353 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.376 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:11.376 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:11.377 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [1ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [1ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@553745510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@40ea4dda -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d1ca1d0 -> wakeup thread Time-limited test)) after lookup
08:04:11.377 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:11.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.378 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.395 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:11.396 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:11.396 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [1ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [1ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [25ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [26ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [26ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [40ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@553745510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@40ea4dda -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1644074626 after retry RPC@1343709656), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d1ca1d0 -> wakeup thread Time-limited test -> (continuation of Deferred@1644074626 after retry RPC@1343709656))) after lookup
08:04:11.397 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:11.398 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.398 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.398 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.415 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:11.415 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:11.416 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=63), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [1ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [1ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [25ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [26ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [26ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [40ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [44ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [46ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [46ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [63ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@553745510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@40ea4dda -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1644074626 after retry RPC@1343709656) -> (continuation of Deferred@472113507 after retry RPC@2001820318), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d1ca1d0 -> wakeup thread Time-limited test -> (continuation of Deferred@1644074626 after retry RPC@1343709656) -> (continuation of Deferred@472113507 after retry RPC@2001820318))) after lookup
08:04:11.416 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:11.417 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.418 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.418 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.457 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:11.457 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:11.457 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=105), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [1ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [1ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [25ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [26ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [26ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [40ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [44ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [46ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [46ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [63ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [64ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [66ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [66ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [104ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@553745510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@40ea4dda -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1644074626 after retry RPC@1343709656) -> (continuation of Deferred@472113507 after retry RPC@2001820318) -> (continuation of Deferred@2007469653 after retry RPC@182435733), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d1ca1d0 -> wakeup thread Time-limited test -> (continuation of Deferred@1644074626 after retry RPC@1343709656) -> (continuation of Deferred@472113507 after retry RPC@2001820318) -> (continuation of Deferred@2007469653 after retry RPC@182435733))) after lookup
08:04:11.458 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:11.460 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.460 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.460 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.776 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:11.776 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:11.777 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=425), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [1ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [1ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [25ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [26ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [26ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [40ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [44ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [46ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [46ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [63ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [64ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [66ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [66ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [104ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [106ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [108ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [108ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [420ms] refreshing cache from master, [420ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [424ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@553745510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@40ea4dda -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1644074626 after retry RPC@1343709656) -> (continuation of Deferred@472113507 after retry RPC@2001820318) -> (continuation of Deferred@2007469653 after retry RPC@182435733) -> (continuation of Deferred@450956797 after retry RPC@1832200739), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d1ca1d0 -> wakeup thread Time-limited test -> (continuation of Deferred@1644074626 after retry RPC@1343709656) -> (continuation of Deferred@472113507 after retry RPC@2001820318) -> (continuation of Deferred@2007469653 after retry RPC@182435733) -> (continuation of Deferred@450956797 after retry RPC@1832200739))) after lookup
08:04:11.778 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:11.779 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.779 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:11.779 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:12.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.321498 22656 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:04:12.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.492854 22667 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:12.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.493288 22667 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:12.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.495465 22667 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:12.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:12.499468 22543 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:04:12.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:12.503319 22543 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:04:12.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.506002 22433 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076" is_pre_election: true
08:04:12.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.507372 22541 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d31321473b59420090c29f4de9f6f928; no voters: e269425272f249e496b7c12b18f63076, e8aea6cb22324b45b188c416f3d49634
08:04:12.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.508141 22667 raft_consensus.cc:2749] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
08:04:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.743582 22672 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader e8aea6cb22324b45b188c416f3d49634)
08:04:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.743956 22672 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.746140 22672 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:12.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:12.749979 22366 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:04:12.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:12.753988 22366 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:04:12.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.758250 22610 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e269425272f249e496b7c12b18f63076" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "d31321473b59420090c29f4de9f6f928" is_pre_election: true
08:04:12.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.758978 22610 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e269425272f249e496b7c12b18f63076 in term 4.
08:04:12.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.760362 22367 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e269425272f249e496b7c12b18f63076; no voters: e8aea6cb22324b45b188c416f3d49634
08:04:12.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.761044 22672 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 5 FOLLOWER]: Leader pre-election won for term 6
08:04:12.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.761304 22672 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 5 FOLLOWER]: Starting leader election (detected failure of leader e8aea6cb22324b45b188c416f3d49634)
08:04:12.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.761663 22672 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 5 FOLLOWER]: Advancing to term 6
08:04:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.766029 22672 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:12.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.767477 22672 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 6 election: Requested vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:12.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.768299 22610 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e269425272f249e496b7c12b18f63076" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "d31321473b59420090c29f4de9f6f928"
08:04:12.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.768740 22610 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 4 FOLLOWER]: Advancing to term 6
08:04:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:12.770571 22366 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:04:12.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.775202 22610 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e269425272f249e496b7c12b18f63076 in term 6.
08:04:12.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.776013 22367 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e269425272f249e496b7c12b18f63076; no voters: e8aea6cb22324b45b188c416f3d49634
08:04:12.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.776603 22672 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 6 FOLLOWER]: Leader election won for term 6
08:04:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.777948 22672 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 6 LEADER]: Becoming Leader. State: Replica: e269425272f249e496b7c12b18f63076, State: Running, Role: LEADER
08:04:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.778769 22672 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:12.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:12.787803 21338 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 reported cstate change: term changed from 5 to 6, leader changed from e8aea6cb22324b45b188c416f3d49634 (127.20.164.65) to e269425272f249e496b7c12b18f63076 (127.20.164.66). New cstate: current_term: 6 leader_uuid: "e269425272f249e496b7c12b18f63076" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: UNKNOWN } } }
08:04:13.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:13.315191 22610 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 6 FOLLOWER]: Refusing update from remote peer e269425272f249e496b7c12b18f63076: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
08:04:13.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:13.316758 22690 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
08:04:13.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:13.324225 22366 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 -> Peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Couldn't send request to peer e8aea6cb22324b45b188c416f3d49634. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:13.456 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:13.457 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:13.457 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2105), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [1ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [1ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [24ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [25ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [26ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [26ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [40ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [44ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [46ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [46ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [63ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [64ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [66ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [66ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [101ms] refreshing cache from master, [101ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [104ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [106ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [108ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [108ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [420ms] refreshing cache from master, [420ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [424ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [425ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [427ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [427ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2101ms] refreshing cache from master, [2101ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [2104ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@553745510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@40ea4dda -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1644074626 after retry RPC@1343709656) -> (continuation of Deferred@472113507 after retry RPC@2001820318) -> (continuation of Deferred@2007469653 after retry RPC@182435733) -> (continuation of Deferred@450956797 after retry RPC@1832200739) -> (continuation of Deferred@1750503150 after retry RPC@754370037), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d1ca1d0 -> wakeup thread Time-limited test -> (continuation of Deferred@1644074626 after retry RPC@1343709656) -> (continuation of Deferred@472113507 after retry RPC@2001820318) -> (continuation of Deferred@2007469653 after retry RPC@182435733) -> (continuation of Deferred@450956797 after retry RPC@1832200739) -> (continuation of Deferred@1750503150 after retry RPC@754370037))) after lookup
08:04:13.458 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:13.459 [DEBUG - kudu-nio-6] (Connection.java:757) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] Successfully connected to peer
08:04:13.484 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x990c770c, L:/127.0.0.1:50238 - R:/127.20.164.66:39351] using TOKEN/null
08:04:13.528 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.164.65:46489
08:04:13.530 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:04:13.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:13.531363 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:13.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:13.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:13.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data
08:04:13.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:13.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/logs
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.65:46489
08:04:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.65
08:04:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33865
08:04:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.65
08:04:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:04:13.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:13.849368 22703 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:13.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:13.849997 22703 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:13.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:13.850783 22703 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:13.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:13.892970 22703 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:13.893815 22703 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.65
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:13.930020 22703 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.65:46489
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.65
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33865
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.65
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/logs
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:13.931373 22703 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:13.932955 22703 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:13.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:13.944289 22711 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:15.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:15.348905 22710 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 22703
08:04:15.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:15.703125 22710 kernel_stack_watchdog.cc:198] Thread 22703 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
08:04:15.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:04:15.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:04:15.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:04:15.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:04:15.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:13.946360 22712 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:15.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:15.704109 22703 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.758s	user 0.001s	sys 0.001s
08:04:15.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:15.704432 22713 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1757 milliseconds
08:04:15.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:15.704445 22703 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.759s	user 0.001s	sys 0.001s
08:04:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.705840 22703 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:15.705896 22714 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:15.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.709411 22703 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:15.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.711356 22703 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.712689 22703 hybrid_clock.cc:648] HybridClock initialized: now 1779091455712649 us; error 33 us; skew 500 ppm
08:04:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.713544 22703 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.719244 22703 webserver.cc:533] Webserver started at http://127.20.164.65:33865/ using document root <none> and password file <none>
08:04:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.720175 22703 fs_manager.cc:362] Metadata directory not provided
08:04:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.720399 22703 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.727669 22703 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.001s
08:04:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.732010 22728 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:04:15.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.732905 22703 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.002s
08:04:15.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.733264 22703 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:15.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e8aea6cb22324b45b188c416f3d49634"
08:04:15.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:44 on dist-test-slave-0rvw"
08:04:15.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.735245 22703 fs_report.cc:389] FS layout report
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/data
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:15.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.790313 22703 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:04:15.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.791786 22703 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:15.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.792299 22703 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:15.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.794824 22703 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:15.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.800259 22735 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
08:04:15.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.808002 22703 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:04:15.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.808198 22703 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.002s	sys 0.000s
08:04:15.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.808506 22703 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:04:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.813328 22703 ts_tablet_manager.cc:616] Registered 1 tablets
08:04:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.813575 22703 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
08:04:15.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.813987 22735 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap starting.
08:04:15.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.877889 22735 log.cc:826] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Log is configured to *not* fsync() on all Append() calls
08:04:15.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.937011 22735 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:15.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.938131 22735 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap complete.
08:04:15.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.939715 22735 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Time spent bootstrapping tablet: real 0.126s	user 0.071s	sys 0.052s
08:04:15.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.957626 22735 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:15.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.960528 22735 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: e8aea6cb22324b45b188c416f3d49634, State: Initialized, Role: FOLLOWER
08:04:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.961573 22735 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:15.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.965027 22735 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Time spent starting tablet: real 0.025s	user 0.017s	sys 0.008s
08:04:15.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.993782 22703 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.65:46489
08:04:15.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.994028 22843 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.65:46489 every 8 connection(s)
08:04:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:15.996312 22703 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:04:16.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.000835 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 22703
08:04:16.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:16.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.038796 22845 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:04:16.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.039330 22845 heartbeater.cc:461] Registering TS with master...
08:04:16.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.040846 22845 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:04:16.049 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:04:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.046476 21257 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:16.049 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.164.66:39351
08:04:16.050 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:04:16.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.050637 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 22338
08:04:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.056902 22844 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:04:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.057200 22844 heartbeater.cc:461] Registering TS with master...
08:04:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.058079 22844 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:04:16.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.061535 21338 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:16.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.063648 21338 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.65:54027
08:04:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.072620 22846 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:04:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.072995 22846 heartbeater.cc:461] Registering TS with master...
08:04:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.073704 22846 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:04:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:16.077324 21182 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:16.092 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:16.092 [DEBUG - kudu-nio-6] (Connection.java:239) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] handling channelInactive
08:04:16.092 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state READY due to: connection closed
08:04:16.093 [DEBUG - Time-limited test] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:16.094 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.094 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.094 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.116 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:16.116 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:16.116 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [2ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [2ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@2140332139(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@65d15b01 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@250967c -> wakeup thread Time-limited test)) after lookup
08:04:16.117 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:16.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.118 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.135 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:16.135 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:16.135 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [2ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [2ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e269425272f249e496b7c12b18f63076, [26ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@2140332139(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@65d15b01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@250967c -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315))) after lookup
08:04:16.136 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:16.136 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.137 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.137 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.176 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:16.176 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:16.177 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [2ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [2ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e269425272f249e496b7c12b18f63076, [26ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [44ms] sending RPC to server e269425272f249e496b7c12b18f63076, [45ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [45ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [84ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@2140332139(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@65d15b01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315) -> (continuation of Deferred@1288222332 after retry RPC@2037983826), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@250967c -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315) -> (continuation of Deferred@1288222332 after retry RPC@2037983826))) after lookup
08:04:16.177 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:16.179 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.179 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.179 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.297 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:16.297 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:16.297 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=205), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [2ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [2ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e269425272f249e496b7c12b18f63076, [26ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [44ms] sending RPC to server e269425272f249e496b7c12b18f63076, [45ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [45ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [84ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [85ms] sending RPC to server e269425272f249e496b7c12b18f63076, [87ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [87ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [201ms] refreshing cache from master, [201ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [204ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@2140332139(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@65d15b01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315) -> (continuation of Deferred@1288222332 after retry RPC@2037983826) -> (continuation of Deferred@665187238 after retry RPC@398358202), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@250967c -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315) -> (continuation of Deferred@1288222332 after retry RPC@2037983826) -> (continuation of Deferred@665187238 after retry RPC@398358202))) after lookup
08:04:16.298 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:16.299 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.299 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.299 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.556 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:16.557 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:16.557 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=464), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [2ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [2ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e269425272f249e496b7c12b18f63076, [26ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [44ms] sending RPC to server e269425272f249e496b7c12b18f63076, [45ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [45ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [84ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [85ms] sending RPC to server e269425272f249e496b7c12b18f63076, [87ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [87ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [201ms] refreshing cache from master, [201ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [204ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [206ms] sending RPC to server e269425272f249e496b7c12b18f63076, [207ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [207ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [460ms] refreshing cache from master, [461ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [464ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@2140332139(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@65d15b01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315) -> (continuation of Deferred@1288222332 after retry RPC@2037983826) -> (continuation of Deferred@665187238 after retry RPC@398358202) -> (continuation of Deferred@785242236 after retry RPC@1252159673), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@250967c -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315) -> (continuation of Deferred@1288222332 after retry RPC@2037983826) -> (continuation of Deferred@665187238 after retry RPC@398358202) -> (continuation of Deferred@785242236 after retry RPC@1252159673))) after lookup
08:04:16.558 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:16.559 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.559 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.559 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.856 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:16.856 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:16.857 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=764), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [2ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [2ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e269425272f249e496b7c12b18f63076, [26ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [44ms] sending RPC to server e269425272f249e496b7c12b18f63076, [45ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [45ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [84ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [85ms] sending RPC to server e269425272f249e496b7c12b18f63076, [87ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [87ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [201ms] refreshing cache from master, [201ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [204ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [206ms] sending RPC to server e269425272f249e496b7c12b18f63076, [207ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [207ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [460ms] refreshing cache from master, [461ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [464ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [466ms] sending RPC to server e269425272f249e496b7c12b18f63076, [467ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [467ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [761ms] refreshing cache from master, [761ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [764ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@2140332139(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@65d15b01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315) -> (continuation of Deferred@1288222332 after retry RPC@2037983826) -> (continuation of Deferred@665187238 after retry RPC@398358202) -> (continuation of Deferred@785242236 after retry RPC@1252159673) -> (continuation of Deferred@1904816209 after retry RPC@687317345), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@250967c -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315) -> (continuation of Deferred@1288222332 after retry RPC@2037983826) -> (continuation of Deferred@665187238 after retry RPC@398358202) -> (continuation of Deferred@785242236 after retry RPC@1252159673) -> (continuation of Deferred@1904816209 after retry RPC@687317345))) after lookup
08:04:16.858 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:16.859 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.859 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:16.859 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:17.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.072849 22844 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:04:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.195851 22855 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader e269425272f249e496b7c12b18f63076)
08:04:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.196218 22855 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:17.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.197686 22855 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:17.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:17.202834 22541 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111)
08:04:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.208324 22798 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "e8aea6cb22324b45b188c416f3d49634" is_pre_election: true
08:04:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.208979 22798 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d31321473b59420090c29f4de9f6f928 in term 5.
08:04:17.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.209995 22543 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e8aea6cb22324b45b188c416f3d49634; no voters: e269425272f249e496b7c12b18f63076
08:04:17.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.210592 22855 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 6 FOLLOWER]: Leader pre-election won for term 7
08:04:17.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.210811 22855 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 6 FOLLOWER]: Starting leader election (detected failure of leader e269425272f249e496b7c12b18f63076)
08:04:17.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.211050 22855 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 6 FOLLOWER]: Advancing to term 7
08:04:17.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.215407 22855 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:17.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.216781 22855 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 7 election: Requested vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.218022 22798 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "e8aea6cb22324b45b188c416f3d49634"
08:04:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.218580 22798 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 5 FOLLOWER]: Advancing to term 7
08:04:17.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:17.221858 22541 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111)
08:04:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.226815 22798 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d31321473b59420090c29f4de9f6f928 in term 7.
08:04:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.227710 22543 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e8aea6cb22324b45b188c416f3d49634; no voters: e269425272f249e496b7c12b18f63076
08:04:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.228255 22855 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 7 FOLLOWER]: Leader election won for term 7
08:04:17.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.229316 22855 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 7 LEADER]: Becoming Leader. State: Replica: d31321473b59420090c29f4de9f6f928, State: Running, Role: LEADER
08:04:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.230330 22855 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:17.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.238078 21338 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 reported cstate change: term changed from 6 to 7, leader changed from e269425272f249e496b7c12b18f63076 (127.20.164.66) to d31321473b59420090c29f4de9f6f928 (127.20.164.67). New cstate: current_term: 7 leader_uuid: "d31321473b59420090c29f4de9f6f928" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: UNKNOWN } } }
08:04:17.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.810359 22798 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 7 FOLLOWER]: Refusing update from remote peer d31321473b59420090c29f4de9f6f928: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
08:04:17.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.811761 22873 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
08:04:17.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:17.829667 22875 mvcc.cc:204] Tried to move back new op lower bound from 7287158593487667200 to 7287158590588502016. Current Snapshot: MvccSnapshot[applied={T|T < 7287158593487667200}]
08:04:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:17.852504 22541 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111) [suppressed 2 similar messages]
08:04:17.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:17.860519 22541 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 -> Peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Couldn't send request to peer e269425272f249e496b7c12b18f63076. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:20.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:20.240500 22541 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 -> Peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Couldn't send request to peer e269425272f249e496b7c12b18f63076. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:04:20.597 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:20.597 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:20.597 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=4505), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [2ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [2ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e269425272f249e496b7c12b18f63076, [26ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [44ms] sending RPC to server e269425272f249e496b7c12b18f63076, [45ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [45ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [84ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [85ms] sending RPC to server e269425272f249e496b7c12b18f63076, [87ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [87ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [201ms] refreshing cache from master, [201ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [204ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [206ms] sending RPC to server e269425272f249e496b7c12b18f63076, [207ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [207ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [460ms] refreshing cache from master, [461ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [464ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [466ms] sending RPC to server e269425272f249e496b7c12b18f63076, [467ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [467ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [761ms] refreshing cache from master, [761ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [764ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [765ms] sending RPC to server e269425272f249e496b7c12b18f63076, [767ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [767ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [4501ms] refreshing cache from master, [4501ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [4504ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@2140332139(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@65d15b01 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315) -> (continuation of Deferred@1288222332 after retry RPC@2037983826) -> (continuation of Deferred@665187238 after retry RPC@398358202) -> (continuation of Deferred@785242236 after retry RPC@1252159673) -> (continuation of Deferred@1904816209 after retry RPC@687317345) -> (continuation of Deferred@1027165628 after retry RPC@676339667), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@250967c -> wakeup thread Time-limited test -> (continuation of Deferred@1596114101 after retry RPC@1497844315) -> (continuation of Deferred@1288222332 after retry RPC@2037983826) -> (continuation of Deferred@665187238 after retry RPC@398358202) -> (continuation of Deferred@785242236 after retry RPC@1252159673) -> (continuation of Deferred@1904816209 after retry RPC@687317345) -> (continuation of Deferred@1027165628 after retry RPC@676339667))) after lookup
08:04:20.598 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:20.599 [DEBUG - kudu-nio-6] (Connection.java:757) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] Successfully connected to peer
08:04:20.619 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x5fc1c7bb, L:/127.0.0.1:50180 - R:/127.20.164.67:37251] using TOKEN/null
08:04:20.663 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.164.66:39351
08:04:20.663 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:04:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:20.664644 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/logs
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.66:39351
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.66
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35829
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.66
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:04:20.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:20.981860 22900 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:20.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:20.982308 22900 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:20.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:20.982729 22900 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:21.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:21.035408 22900 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:21.036777 22900 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.66
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:21.078696 22900 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.66:39351
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.66
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35829
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.66
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/logs
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:21.080070 22900 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:21.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:21.081575 22900 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:21.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:21.093528 22906 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:22.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:21.094573 22907 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:22.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:22.480616 22900 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.387s	user 0.457s	sys 0.867s
08:04:22.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:22.480962 22900 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.387s	user 0.457s	sys 0.867s
08:04:22.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:22.481115 22908 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1382 milliseconds
08:04:22.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:22.481415 22909 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:22.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.482273 22900 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:22.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.483481 22900 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:22.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.485533 22900 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.486900 22900 hybrid_clock.cc:648] HybridClock initialized: now 1779091462486852 us; error 52 us; skew 500 ppm
08:04:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.487723 22900 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:22.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.495406 22900 webserver.cc:533] Webserver started at http://127.20.164.66:35829/ using document root <none> and password file <none>
08:04:22.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.496397 22900 fs_manager.cc:362] Metadata directory not provided
08:04:22.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.496622 22900 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:22.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.504719 22900 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.002s
08:04:22.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.509788 22919 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:04:22.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.511152 22900 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.003s
08:04:22.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.511533 22900 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:22.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e269425272f249e496b7c12b18f63076"
08:04:22.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:47 on dist-test-slave-0rvw"
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.513484 22900 fs_report.cc:389] FS layout report
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/data
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:22.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:22.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.596917 22900 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:04:22.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.598469 22900 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:22.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.598976 22900 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:22.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:22.601536 22541 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 -> Peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Couldn't send request to peer e269425272f249e496b7c12b18f63076. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:04:22.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.602326 22900 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:22.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.608107 22928 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
08:04:22.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.616182 22900 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:04:22.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.616391 22900 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
08:04:22.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.616607 22900 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:04:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.621896 22900 ts_tablet_manager.cc:616] Registered 1 tablets
08:04:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.622135 22900 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.003s	sys 0.000s
08:04:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.622525 22928 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Bootstrap starting.
08:04:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.697881 22928 log.cc:826] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Log is configured to *not* fsync() on all Append() calls
08:04:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.767825 22928 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.768741 22928 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Bootstrap complete.
08:04:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.770332 22928 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Time spent bootstrapping tablet: real 0.148s	user 0.097s	sys 0.047s
08:04:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.787755 22928 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.789937 22928 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: e269425272f249e496b7c12b18f63076, State: Initialized, Role: FOLLOWER
08:04:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.790841 22928 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:22.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.795892 22928 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Time spent starting tablet: real 0.025s	user 0.025s	sys 0.000s
08:04:22.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.807441 22900 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.66:39351
08:04:22.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.807742 23036 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.66:39351 every 8 connection(s)
08:04:22.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.810964 22900 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:04:22.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.819177 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 22900
08:04:22.820 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.856053 23038 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:04:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.856746 23038 heartbeater.cc:461] Registering TS with master...
08:04:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.858243 23038 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:04:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.863310 21258 ts_manager.cc:194] Re-registered known tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:22.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.865715 23037 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:04:22.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.866012 23037 heartbeater.cc:461] Registering TS with master...
08:04:22.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.867203 23037 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:04:22.869 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:04:22.870 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.164.67:37251
08:04:22.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:04:22.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.870240 21337 ts_manager.cc:194] Re-registered known tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:22.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.871325 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 22522
08:04:22.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.873443 21337 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.66:49455
08:04:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.875963 23039 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:04:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.876271 23039 heartbeater.cc:461] Registering TS with master...
08:04:22.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.877322 23039 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:04:22.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:22.880863 21182 ts_manager.cc:194] Re-registered known tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:22.902 [DEBUG - kudu-nio-6] (Connection.java:239) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] handling channelInactive
08:04:22.902 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state READY due to: connection closed
08:04:22.903 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:22.904 [DEBUG - Time-limited test] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:22.905 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.905 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.906 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.916 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:22.916 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:22.916 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@724963079(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@3b02335 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f29f1bd -> wakeup thread Time-limited test)) after lookup
08:04:22.917 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:22.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.918 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.936 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:22.936 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:22.936 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [13ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [14ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [14ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@724963079(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@3b02335 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f29f1bd -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730))) after lookup
08:04:22.937 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:22.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.938 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.975 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:22.976 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:22.976 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [13ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [14ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [14ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [33ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [34ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [34ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@724963079(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@3b02335 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730) -> (continuation of Deferred@1913271070 after retry RPC@755390774), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f29f1bd -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730) -> (continuation of Deferred@1913271070 after retry RPC@755390774))) after lookup
08:04:22.977 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:22.977 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.978 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.978 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.997 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:22.997 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:22.997 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=93), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [13ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [14ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [14ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [33ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [34ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [34ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [72ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [74ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [74ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [92ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@724963079(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@3b02335 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730) -> (continuation of Deferred@1913271070 after retry RPC@755390774) -> (continuation of Deferred@838005172 after retry RPC@782422357), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f29f1bd -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730) -> (continuation of Deferred@1913271070 after retry RPC@755390774) -> (continuation of Deferred@838005172 after retry RPC@782422357))) after lookup
08:04:22.998 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:22.999 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.999 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:22.999 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:23.136 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:23.136 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:23.137 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=233), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [13ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [14ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [14ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [33ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [34ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [34ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [72ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [74ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [74ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [92ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [94ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [95ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [95ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [228ms] refreshing cache from master, [228ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [232ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@724963079(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@3b02335 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730) -> (continuation of Deferred@1913271070 after retry RPC@755390774) -> (continuation of Deferred@838005172 after retry RPC@782422357) -> (continuation of Deferred@2093431769 after retry RPC@1146578285), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f29f1bd -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730) -> (continuation of Deferred@1913271070 after retry RPC@755390774) -> (continuation of Deferred@838005172 after retry RPC@782422357) -> (continuation of Deferred@2093431769 after retry RPC@1146578285))) after lookup
08:04:23.137 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:23.138 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:23.138 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:23.138 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:23.456 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:23.456 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:23.457 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=553), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [13ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [14ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [14ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [33ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [34ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [34ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [72ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [74ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [74ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [92ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [94ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [95ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [95ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [228ms] refreshing cache from master, [228ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [232ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [233ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [234ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [235ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [549ms] refreshing cache from master, [549ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [552ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@724963079(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@3b02335 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730) -> (continuation of Deferred@1913271070 after retry RPC@755390774) -> (continuation of Deferred@838005172 after retry RPC@782422357) -> (continuation of Deferred@2093431769 after retry RPC@1146578285) -> (continuation of Deferred@1705484492 after retry RPC@447008024), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f29f1bd -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730) -> (continuation of Deferred@1913271070 after retry RPC@755390774) -> (continuation of Deferred@838005172 after retry RPC@782422357) -> (continuation of Deferred@2093431769 after retry RPC@1146578285) -> (continuation of Deferred@1705484492 after retry RPC@447008024))) after lookup
08:04:23.458 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:23.458 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:23.459 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state CONNECTING due to: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:23.459 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d31321473b59420090c29f4de9f6f928(127.20.164.67:37251) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251
08:04:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:23.878046 23037 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:04:24.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.163302 23049 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader d31321473b59420090c29f4de9f6f928)
08:04:24.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.163976 23049 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:24.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.166368 23049 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:24.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:24.169961 22732 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:04:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:24.173818 22732 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:04:24.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.179316 22991 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e8aea6cb22324b45b188c416f3d49634" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076" is_pre_election: true
08:04:24.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.179968 22991 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e8aea6cb22324b45b188c416f3d49634 in term 6.
08:04:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.181085 22729 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e269425272f249e496b7c12b18f63076, e8aea6cb22324b45b188c416f3d49634; no voters: d31321473b59420090c29f4de9f6f928
08:04:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.181671 23049 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 7 FOLLOWER]: Leader pre-election won for term 8
08:04:24.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.181931 23049 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 7 FOLLOWER]: Starting leader election (detected failure of leader d31321473b59420090c29f4de9f6f928)
08:04:24.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.182214 23049 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 7 FOLLOWER]: Advancing to term 8
08:04:24.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.186805 23049 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.188198 23049 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 8 election: Requested vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.189090 22991 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e8aea6cb22324b45b188c416f3d49634" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076"
08:04:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.189568 22991 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 6 FOLLOWER]: Advancing to term 8
08:04:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:24.190804 22732 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111)
08:04:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.195848 22991 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e8aea6cb22324b45b188c416f3d49634 in term 8.
08:04:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.196611 22729 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e269425272f249e496b7c12b18f63076, e8aea6cb22324b45b188c416f3d49634; no voters: d31321473b59420090c29f4de9f6f928
08:04:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.197105 23049 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 8 FOLLOWER]: Leader election won for term 8
08:04:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.198493 23049 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 8 LEADER]: Becoming Leader. State: Replica: e8aea6cb22324b45b188c416f3d49634, State: Running, Role: LEADER
08:04:24.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.199431 23049 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:24.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.208316 21337 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 reported cstate change: term changed from 7 to 8, leader changed from d31321473b59420090c29f4de9f6f928 (127.20.164.67) to e8aea6cb22324b45b188c416f3d49634 (127.20.164.65). New cstate: current_term: 8 leader_uuid: "e8aea6cb22324b45b188c416f3d49634" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: HEALTHY } } }
08:04:24.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.656843 22991 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 8 FOLLOWER]: Refusing update from remote peer e8aea6cb22324b45b188c416f3d49634: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
08:04:24.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.658349 23049 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
08:04:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:24.672915 23068 mvcc.cc:204] Tried to move back new op lower bound from 7287158622716416000 to 7287158608822398976. Current Snapshot: MvccSnapshot[applied={T|T < 7287158622716416000}]
08:04:24.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:24.688541 22732 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 -> Peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Couldn't send request to peer d31321473b59420090c29f4de9f6f928. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:25.956 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:25.957 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:25.957 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=3053), Traces: [0ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [2ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [2ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [11ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [13ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [14ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [14ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [33ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [34ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [34ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [72ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [74ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [74ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [92ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [94ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [95ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [95ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [228ms] refreshing cache from master, [228ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [232ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [233ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [234ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [235ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [549ms] refreshing cache from master, [549ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [552ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [553ms] sending RPC to server d31321473b59420090c29f4de9f6f928, [555ms] received response from server d31321473b59420090c29f4de9f6f928: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [555ms] delaying RPC due to: Network error: Failed to connect to peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251): Connection refused: /127.20.164.67:37251, [3048ms] refreshing cache from master, [3048ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [3052ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@724963079(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@3b02335 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730) -> (continuation of Deferred@1913271070 after retry RPC@755390774) -> (continuation of Deferred@838005172 after retry RPC@782422357) -> (continuation of Deferred@2093431769 after retry RPC@1146578285) -> (continuation of Deferred@1705484492 after retry RPC@447008024) -> (continuation of Deferred@1423629226 after retry RPC@1755975716), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f29f1bd -> wakeup thread Time-limited test -> (continuation of Deferred@895321829 after retry RPC@636042730) -> (continuation of Deferred@1913271070 after retry RPC@755390774) -> (continuation of Deferred@838005172 after retry RPC@782422357) -> (continuation of Deferred@2093431769 after retry RPC@1146578285) -> (continuation of Deferred@1705484492 after retry RPC@447008024) -> (continuation of Deferred@1423629226 after retry RPC@1755975716))) after lookup
08:04:25.958 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:25.959 [DEBUG - kudu-nio-6] (Connection.java:757) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] Successfully connected to peer
08:04:25.980 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x521b4452, L:/127.0.0.1:47228 - R:/127.20.164.65:46489] using TOKEN/null
08:04:26.038 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.164.67:37251
08:04:26.039 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:26.041270 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/logs
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.67:37251
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.67
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42487
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.67
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:04:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:26.357666 23090 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:26.358184 23090 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:26.358697 23090 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:26.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:26.392544 23090 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:26.393440 23090 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.67
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:26.429811 23090 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.67:37251
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.67
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42487
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.67
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/logs
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:04:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:04:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:04:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:26.431032 23090 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:26.432641 23090 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:26.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:26.445174 23096 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:27.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:27.208617 22732 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 -> Peer d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251): Couldn't send request to peer d31321473b59420090c29f4de9f6f928. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.67:37251: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:04:28.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:26.446171 23097 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:28.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:28.149468 23090 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.704s	user 0.567s	sys 1.086s
08:04:28.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:27.850731 23095 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 23090
08:04:28.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:28.152630 23090 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.707s	user 0.567s	sys 1.086s
08:04:28.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:28.151579 23099 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:28.156785 23098 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1704 milliseconds
08:04:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.156785 23090 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:28.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.158330 23090 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.160738 23090 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.162303 23090 hybrid_clock.cc:648] HybridClock initialized: now 1779091468162234 us; error 71 us; skew 500 ppm
08:04:28.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.163035 23090 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:28.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.169325 23090 webserver.cc:533] Webserver started at http://127.20.164.67:42487/ using document root <none> and password file <none>
08:04:28.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.170151 23090 fs_manager.cc:362] Metadata directory not provided
08:04:28.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.170334 23090 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:28.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.177687 23090 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.001s	sys 0.003s
08:04:28.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.182219 23111 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:04:28.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.183323 23090 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
08:04:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.183655 23090 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:04:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d31321473b59420090c29f4de9f6f928"
08:04:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:49 on dist-test-slave-0rvw"
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.185508 23090 fs_report.cc:389] FS layout report
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/wal
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/data
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.244980 23090 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:04:28.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.246480 23090 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:28.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.246974 23090 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:28.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.249917 23090 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:28.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.256004 23118 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
08:04:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.267951 23090 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:04:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.268240 23090 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.014s	user 0.000s	sys 0.002s
08:04:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.268582 23090 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:04:28.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.273505 23090 ts_tablet_manager.cc:616] Registered 1 tablets
08:04:28.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.273730 23090 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.000s	sys 0.003s
08:04:28.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.274183 23118 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Bootstrap starting.
08:04:28.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.352795 23118 log.cc:826] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Log is configured to *not* fsync() on all Append() calls
08:04:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.469259 23118 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:28.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.471033 23118 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Bootstrap complete.
08:04:28.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.473479 23118 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Time spent bootstrapping tablet: real 0.200s	user 0.141s	sys 0.050s
08:04:28.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.481843 23090 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.67:37251
08:04:28.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.481990 23225 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.67:37251 every 8 connection(s)
08:04:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.485208 23090 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-2/data/info.pb
08:04:28.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.494923 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 23090
08:04:28.495 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.540062 23227 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:04:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.540544 23227 heartbeater.cc:461] Registering TS with master...
08:04:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.541714 23227 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:04:28.546 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:04:28.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.546348 21258 ts_manager.cc:194] Re-registered known tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:04:28.547 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.164.65:46489
08:04:28.547 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:04:28.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.548225 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 22703
08:04:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.552834 23118 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:28.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.555788 23118 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: d31321473b59420090c29f4de9f6f928, State: Initialized, Role: FOLLOWER
08:04:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.559597 23226 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:04:28.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.559998 23226 heartbeater.cc:461] Registering TS with master...
08:04:28.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.558859 23118 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:28.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.561254 23226 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:04:28.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.564597 21337 ts_manager.cc:194] Re-registered known tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:04:28.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.568461 21337 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.67:53893
08:04:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.575749 23228 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:04:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.576109 23228 heartbeater.cc:461] Registering TS with master...
08:04:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.576792 23228 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:04:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.580819 21183 ts_manager.cc:194] Re-registered known tserver with Master: d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251)
08:04:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.588200 23226 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:04:28.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:28.588874 23118 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928: Time spent starting tablet: real 0.114s	user 0.026s	sys 0.024s
08:04:28.601 [DEBUG - kudu-nio-6] (Connection.java:239) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] handling channelInactive
08:04:28.601 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:28.601 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state READY due to: connection closed
08:04:28.601 [DEBUG - Time-limited test] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:28.602 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.603 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.603 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.615 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:28.616 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:28.616 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@615705764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@1c21e326 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4eb29ab8 -> wakeup thread Time-limited test)) after lookup
08:04:28.617 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:28.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.618 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.636 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:28.636 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:28.636 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@615705764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@1c21e326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4eb29ab8 -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524))) after lookup
08:04:28.637 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:28.637 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.637 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.637 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.655 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:28.655 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:28.656 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=54), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [35ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [36ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [36ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@615705764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@1c21e326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524) -> (continuation of Deferred@17332067 after retry RPC@116298759), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4eb29ab8 -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524) -> (continuation of Deferred@17332067 after retry RPC@116298759))) after lookup
08:04:28.657 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:28.658 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.658 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.658 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.755 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:28.756 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:28.756 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=154), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [35ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [36ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [36ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [55ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [57ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [57ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [154ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@615705764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@1c21e326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524) -> (continuation of Deferred@17332067 after retry RPC@116298759) -> (continuation of Deferred@1735886203 after retry RPC@365652798), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4eb29ab8 -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524) -> (continuation of Deferred@17332067 after retry RPC@116298759) -> (continuation of Deferred@1735886203 after retry RPC@365652798))) after lookup
08:04:28.756 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:28.757 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.757 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.758 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.975 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:28.976 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:28.976 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=374), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [35ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [36ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [36ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [55ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [57ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [57ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [154ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [155ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [157ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [157ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [371ms] refreshing cache from master, [371ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [374ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@615705764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@1c21e326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524) -> (continuation of Deferred@17332067 after retry RPC@116298759) -> (continuation of Deferred@1735886203 after retry RPC@365652798) -> (continuation of Deferred@35014046 after retry RPC@1279941753), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4eb29ab8 -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524) -> (continuation of Deferred@17332067 after retry RPC@116298759) -> (continuation of Deferred@1735886203 after retry RPC@365652798) -> (continuation of Deferred@35014046 after retry RPC@1279941753))) after lookup
08:04:28.977 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:28.978 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.978 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:28.978 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:29.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.769420 23239 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader e8aea6cb22324b45b188c416f3d49634)
08:04:29.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.769873 23239 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:29.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.772147 23239 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:29.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:29.776027 22922 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:04:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:29.779882 22922 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:04:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.782903 23181 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e269425272f249e496b7c12b18f63076" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "d31321473b59420090c29f4de9f6f928" is_pre_election: true
08:04:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.783577 23181 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e269425272f249e496b7c12b18f63076 in term 7.
08:04:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.784713 22923 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e269425272f249e496b7c12b18f63076; no voters: e8aea6cb22324b45b188c416f3d49634
08:04:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.785394 23239 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 8 FOLLOWER]: Leader pre-election won for term 9
08:04:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.785683 23239 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 8 FOLLOWER]: Starting leader election (detected failure of leader e8aea6cb22324b45b188c416f3d49634)
08:04:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.786003 23239 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 8 FOLLOWER]: Advancing to term 9
08:04:29.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.790287 23239 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:29.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.791638 23239 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 9 election: Requested vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.792403 23181 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e269425272f249e496b7c12b18f63076" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "d31321473b59420090c29f4de9f6f928"
08:04:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.792778 23181 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 7 FOLLOWER]: Advancing to term 9
08:04:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:29.794688 22922 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:04:29.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.799165 23181 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e269425272f249e496b7c12b18f63076 in term 9.
08:04:29.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.799875 22923 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e269425272f249e496b7c12b18f63076; no voters: e8aea6cb22324b45b188c416f3d49634
08:04:29.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.800419 23239 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 9 FOLLOWER]: Leader election won for term 9
08:04:29.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.801803 23239 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 9 LEADER]: Becoming Leader. State: Replica: e269425272f249e496b7c12b18f63076, State: Running, Role: LEADER
08:04:29.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.802711 23239 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:29.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:29.813232 21337 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 reported cstate change: term changed from 8 to 9, leader changed from e8aea6cb22324b45b188c416f3d49634 (127.20.164.65) to e269425272f249e496b7c12b18f63076 (127.20.164.66). New cstate: current_term: 9 leader_uuid: "e269425272f249e496b7c12b18f63076" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: UNKNOWN } } }
08:04:29.839 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:29.839 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:29.840 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1238), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [35ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [36ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [36ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [55ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [57ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [57ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [154ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [155ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [157ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [157ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [371ms] refreshing cache from master, [371ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [374ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [376ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [377ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [377ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [1232ms] refreshing cache from master, [1232ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [1237ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@615705764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@1c21e326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524) -> (continuation of Deferred@17332067 after retry RPC@116298759) -> (continuation of Deferred@1735886203 after retry RPC@365652798) -> (continuation of Deferred@35014046 after retry RPC@1279941753) -> (continuation of Deferred@334716991 after retry RPC@222580652), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4eb29ab8 -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524) -> (continuation of Deferred@17332067 after retry RPC@116298759) -> (continuation of Deferred@1735886203 after retry RPC@365652798) -> (continuation of Deferred@35014046 after retry RPC@1279941753) -> (continuation of Deferred@334716991 after retry RPC@222580652))) after lookup
08:04:29.841 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:29.848 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:29.849 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:29.849 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:30.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:30.319355 22922 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 -> Peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Couldn't send request to peer e8aea6cb22324b45b188c416f3d49634. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:30.326012 23181 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 9 FOLLOWER]: Refusing update from remote peer e269425272f249e496b7c12b18f63076: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
08:04:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:30.327561 23258 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
08:04:31.556 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:31.556 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:31.557 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2955), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [16ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [17ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [17ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [34ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [35ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [36ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [36ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [54ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [55ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [57ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [57ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [154ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [155ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [157ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [157ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [371ms] refreshing cache from master, [371ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [374ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [376ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [377ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [377ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [1232ms] refreshing cache from master, [1232ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [1237ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [1240ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [1248ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [1248ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2952ms] refreshing cache from master, [2952ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [2955ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@615705764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@1c21e326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524) -> (continuation of Deferred@17332067 after retry RPC@116298759) -> (continuation of Deferred@1735886203 after retry RPC@365652798) -> (continuation of Deferred@35014046 after retry RPC@1279941753) -> (continuation of Deferred@334716991 after retry RPC@222580652) -> (continuation of Deferred@374068708 after retry RPC@873513457), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4eb29ab8 -> wakeup thread Time-limited test -> (continuation of Deferred@1945439408 after retry RPC@692596524) -> (continuation of Deferred@17332067 after retry RPC@116298759) -> (continuation of Deferred@1735886203 after retry RPC@365652798) -> (continuation of Deferred@35014046 after retry RPC@1279941753) -> (continuation of Deferred@334716991 after retry RPC@222580652) -> (continuation of Deferred@374068708 after retry RPC@873513457))) after lookup
08:04:31.557 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:31.558 [DEBUG - kudu-nio-6] (Connection.java:757) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] Successfully connected to peer
08:04:31.577 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x9a9aa4b0, L:/127.0.0.1:57994 - R:/127.20.164.66:39351] using TOKEN/null
08:04:31.618 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.164.65:46489
08:04:31.618 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:31.620045 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/logs
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.65:46489
08:04:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.65
08:04:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33865
08:04:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.65
08:04:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:04:31.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:31.939555 23280 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:31.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:31.940081 23280 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:31.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:31.940701 23280 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:31.973836 23280 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:31.974761 23280 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.65
08:04:32.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:32.010279 23280 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:32.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:32.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:32.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:32.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:32.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data
08:04:32.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.65:46489
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.65
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33865
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.65
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/logs
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:04:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:32.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:32.011965 23280 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:32.013609 23280 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:32.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:32.026542 23288 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:32.699416 22922 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 -> Peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Couldn't send request to peer e8aea6cb22324b45b188c416f3d49634. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:04:33.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:32.032884 23291 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:33.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:32.028234 23289 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:33.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:33.360412 23290 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1329 milliseconds
08:04:33.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.360445 23280 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.361683 23280 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:33.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.363948 23280 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.365329 23280 hybrid_clock.cc:648] HybridClock initialized: now 1779091473365278 us; error 55 us; skew 500 ppm
08:04:33.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.366220 23280 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:33.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.372478 23280 webserver.cc:533] Webserver started at http://127.20.164.65:33865/ using document root <none> and password file <none>
08:04:33.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.373479 23280 fs_manager.cc:362] Metadata directory not provided
08:04:33.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.373709 23280 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:33.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.383553 23280 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.002s	sys 0.005s
08:04:33.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.388161 23300 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:04:33.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.389106 23280 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.005s	sys 0.000s
08:04:33.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.389443 23280 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:33.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e8aea6cb22324b45b188c416f3d49634"
08:04:33.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:44 on dist-test-slave-0rvw"
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.391592 23280 fs_report.cc:389] FS layout report
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/data
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:33.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.441591 23280 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:04:33.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.443058 23280 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:33.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.443555 23280 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.446033 23280 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:33.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.451709 23307 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
08:04:33.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.459388 23280 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:04:33.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.459625 23280 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.002s	sys 0.000s
08:04:33.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.459991 23280 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:04:33.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.464443 23280 ts_tablet_manager.cc:616] Registered 1 tablets
08:04:33.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.464735 23280 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
08:04:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.465081 23307 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap starting.
08:04:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.542320 23307 log.cc:826] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Log is configured to *not* fsync() on all Append() calls
08:04:33.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.633647 23307 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:33.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.634464 23307 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap complete.
08:04:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.636387 23307 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Time spent bootstrapping tablet: real 0.172s	user 0.123s	sys 0.046s
08:04:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.651659 23280 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.65:46489
08:04:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.651814 23414 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.65:46489 every 8 connection(s)
08:04:33.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.654174 23280 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:04:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.653946 23307 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.656692 23307 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: e8aea6cb22324b45b188c416f3d49634, State: Initialized, Role: FOLLOWER
08:04:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.657773 23307 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:33.661 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:33.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.660187 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 23280
08:04:33.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.694904 23307 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Time spent starting tablet: real 0.058s	user 0.037s	sys 0.020s
08:04:33.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.720716 23415 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:04:33.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.721228 23415 heartbeater.cc:461] Registering TS with master...
08:04:33.724 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:04:33.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.722466 23415 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:04:33.724 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.164.66:39351
08:04:33.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:04:33.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.725317 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 22900
08:04:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.729085 23370 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 8 FOLLOWER]: Advancing to term 9
08:04:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.730099 23416 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:04:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.730342 23416 heartbeater.cc:461] Registering TS with master...
08:04:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.730923 23416 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:04:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.733318 21337 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.736299 21337 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.65:41307
08:04:33.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.740123 23415 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:04:33.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.742357 23424 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:04:33.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.742677 23424 heartbeater.cc:461] Registering TS with master...
08:04:33.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.743287 23424 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:04:33.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.764607 21258 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:33.765 [DEBUG - kudu-nio-6] (Connection.java:239) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] handling channelInactive
08:04:33.766 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state READY due to: connection closed
08:04:33.766 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:33.767 [DEBUG - Time-limited test] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:33.769 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.769 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.769 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:33.765347 23302 connection.cc:582] server connection from 127.20.164.66:60023 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:33.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:33.767648 21182 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:33.776 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:33.776 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:33.777 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [3ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [3ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1965056049(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@44849323 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@42571878 -> wakeup thread Time-limited test)) after lookup
08:04:33.778 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:33.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.779 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.795 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:33.796 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:33.796 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [3ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [3ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [11ms] sending RPC to server e269425272f249e496b7c12b18f63076, [13ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [13ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1965056049(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@44849323 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2049823135 after retry RPC@12817018), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@42571878 -> wakeup thread Time-limited test -> (continuation of Deferred@2049823135 after retry RPC@12817018))) after lookup
08:04:33.796 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:33.797 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.797 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.797 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.816 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:33.816 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:33.816 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=50), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [3ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [3ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [11ms] sending RPC to server e269425272f249e496b7c12b18f63076, [13ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [13ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [30ms] sending RPC to server e269425272f249e496b7c12b18f63076, [31ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [31ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [50ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1965056049(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@44849323 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2049823135 after retry RPC@12817018) -> (continuation of Deferred@1028184031 after retry RPC@42269210), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@42571878 -> wakeup thread Time-limited test -> (continuation of Deferred@2049823135 after retry RPC@12817018) -> (continuation of Deferred@1028184031 after retry RPC@42269210))) after lookup
08:04:33.817 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:33.819 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.819 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.819 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.876 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:33.876 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:33.877 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=110), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [3ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [3ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [11ms] sending RPC to server e269425272f249e496b7c12b18f63076, [13ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [13ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [30ms] sending RPC to server e269425272f249e496b7c12b18f63076, [31ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [31ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [50ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [51ms] sending RPC to server e269425272f249e496b7c12b18f63076, [53ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [53ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [106ms] refreshing cache from master, [107ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [110ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1965056049(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@44849323 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2049823135 after retry RPC@12817018) -> (continuation of Deferred@1028184031 after retry RPC@42269210) -> (continuation of Deferred@99921132 after retry RPC@537252450), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@42571878 -> wakeup thread Time-limited test -> (continuation of Deferred@2049823135 after retry RPC@12817018) -> (continuation of Deferred@1028184031 after retry RPC@42269210) -> (continuation of Deferred@99921132 after retry RPC@537252450))) after lookup
08:04:33.877 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:33.878 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.878 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.878 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.956 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:33.956 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351)[L],e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:33.957 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=190), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [3ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [3ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [11ms] sending RPC to server e269425272f249e496b7c12b18f63076, [13ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [13ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [30ms] sending RPC to server e269425272f249e496b7c12b18f63076, [31ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [31ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [50ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [51ms] sending RPC to server e269425272f249e496b7c12b18f63076, [53ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [53ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [106ms] refreshing cache from master, [107ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [110ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [111ms] sending RPC to server e269425272f249e496b7c12b18f63076, [112ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [113ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [186ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [190ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1965056049(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@44849323 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2049823135 after retry RPC@12817018) -> (continuation of Deferred@1028184031 after retry RPC@42269210) -> (continuation of Deferred@99921132 after retry RPC@537252450) -> (continuation of Deferred@1906999703 after retry RPC@2142427906), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@42571878 -> wakeup thread Time-limited test -> (continuation of Deferred@2049823135 after retry RPC@12817018) -> (continuation of Deferred@1028184031 after retry RPC@42269210) -> (continuation of Deferred@99921132 after retry RPC@537252450) -> (continuation of Deferred@1906999703 after retry RPC@2142427906))) after lookup
08:04:33.958 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] connecting to peer
08:04:33.959 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.959 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] cleaning up while in state CONNECTING due to: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:33.959 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e269425272f249e496b7c12b18f63076(127.20.164.66:39351) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351
08:04:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.096503 23436 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader e269425272f249e496b7c12b18f63076)
08:04:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.096858 23436 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:35.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.098768 23436 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:35.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.101568 23112 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111)
08:04:35.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.105137 23112 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111)
08:04:35.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.108825 23370 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "e8aea6cb22324b45b188c416f3d49634" is_pre_election: true
08:04:35.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.110239 23114 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d31321473b59420090c29f4de9f6f928; no voters: e269425272f249e496b7c12b18f63076, e8aea6cb22324b45b188c416f3d49634
08:04:35.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.110904 23436 raft_consensus.cc:2749] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
08:04:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.240157 23442 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader e269425272f249e496b7c12b18f63076)
08:04:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.240674 23442 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:35.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.243229 23442 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:35.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.247464 23301 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111)
08:04:35.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.251436 23301 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111)
08:04:35.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.253352 23181 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e8aea6cb22324b45b188c416f3d49634" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "d31321473b59420090c29f4de9f6f928" is_pre_election: true
08:04:35.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.253803 23181 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e8aea6cb22324b45b188c416f3d49634 in term 9.
08:04:35.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.254704 23304 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e8aea6cb22324b45b188c416f3d49634; no voters: e269425272f249e496b7c12b18f63076
08:04:35.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.255314 23442 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 9 FOLLOWER]: Leader pre-election won for term 10
08:04:35.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.255535 23442 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 9 FOLLOWER]: Starting leader election (detected failure of leader e269425272f249e496b7c12b18f63076)
08:04:35.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.255729 23442 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 9 FOLLOWER]: Advancing to term 10
08:04:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.260074 23442 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.261384 23442 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 10 election: Requested vote from peers d31321473b59420090c29f4de9f6f928 (127.20.164.67:37251), e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.262049 23181 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "e8aea6cb22324b45b188c416f3d49634" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "d31321473b59420090c29f4de9f6f928"
08:04:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.262499 23181 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 9 FOLLOWER]: Advancing to term 10
08:04:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.264405 23301 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111)
08:04:35.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.266661 23181 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e8aea6cb22324b45b188c416f3d49634 in term 10.
08:04:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.267517 23304 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e8aea6cb22324b45b188c416f3d49634; no voters: e269425272f249e496b7c12b18f63076
08:04:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.268021 23442 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 10 FOLLOWER]: Leader election won for term 10
08:04:35.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.269490 23442 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 10 LEADER]: Becoming Leader. State: Replica: e8aea6cb22324b45b188c416f3d49634, State: Running, Role: LEADER
08:04:35.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.270308 23442 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:35.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.279589 21338 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 reported cstate change: term changed from 9 to 10, leader changed from e269425272f249e496b7c12b18f63076 (127.20.164.66) to e8aea6cb22324b45b188c416f3d49634 (127.20.164.65). New cstate: current_term: 10 leader_uuid: "e8aea6cb22324b45b188c416f3d49634" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: HEALTHY } } }
08:04:35.516 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:35.516 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:35.516 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1750), Traces: [0ms] sending RPC to server e269425272f249e496b7c12b18f63076, [3ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [3ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [11ms] sending RPC to server e269425272f249e496b7c12b18f63076, [13ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [13ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [29ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [30ms] sending RPC to server e269425272f249e496b7c12b18f63076, [31ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [31ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [50ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [51ms] sending RPC to server e269425272f249e496b7c12b18f63076, [53ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [53ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [106ms] refreshing cache from master, [107ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [110ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [111ms] sending RPC to server e269425272f249e496b7c12b18f63076, [112ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [113ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [186ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [190ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [192ms] sending RPC to server e269425272f249e496b7c12b18f63076, [193ms] received response from server e269425272f249e496b7c12b18f63076: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [193ms] delaying RPC due to: Network error: Failed to connect to peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351): Connection refused: /127.20.164.66:39351, [1747ms] refreshing cache from master, [1747ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [1750ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1965056049(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@44849323 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2049823135 after retry RPC@12817018) -> (continuation of Deferred@1028184031 after retry RPC@42269210) -> (continuation of Deferred@99921132 after retry RPC@537252450) -> (continuation of Deferred@1906999703 after retry RPC@2142427906) -> (continuation of Deferred@960594646 after retry RPC@1346511927), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@42571878 -> wakeup thread Time-limited test -> (continuation of Deferred@2049823135 after retry RPC@12817018) -> (continuation of Deferred@1028184031 after retry RPC@42269210) -> (continuation of Deferred@99921132 after retry RPC@537252450) -> (continuation of Deferred@1906999703 after retry RPC@2142427906) -> (continuation of Deferred@960594646 after retry RPC@1346511927))) after lookup
08:04:35.517 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:35.518 [DEBUG - kudu-nio-5] (Connection.java:757) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] Successfully connected to peer
08:04:35.539 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0xf89f9376, L:/127.0.0.1:33360 - R:/127.20.164.65:46489] using TOKEN/null
08:04:35.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.547943 23181 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 10 FOLLOWER]: Refusing update from remote peer e8aea6cb22324b45b188c416f3d49634: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 10 index: 38. (index mismatch)
08:04:35.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.549371 23447 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 36, Time since last communication: 0.001s
08:04:35.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.550173 23301 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 -> Peer e269425272f249e496b7c12b18f63076 (127.20.164.66:39351): Couldn't send request to peer e269425272f249e496b7c12b18f63076. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.66:39351: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:35.597 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.164.66:39351
08:04:35.598 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:35.599689 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/logs
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.66:39351
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.66
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35829
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.66
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:04:35.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.934686 23466 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:35.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.935267 23466 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.935875 23466 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:35.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.968009 23466 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:35.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:35.968951 23466 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.66
08:04:36.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:36.004427 23466 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:36.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.66:39351
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.66
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35829
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.66
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/logs
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:04:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:04:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:04:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:04:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:36.006145 23466 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:36.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:36.008074 23466 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:36.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:36.022279 23472 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:37.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:37.153165 21289 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5840 after lost signal to thread 21224
08:04:37.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:37.153820 21289 debug-util.cc:398] Leaking SignalData structure 0x7b08000957e0 after lost signal to thread 21292
08:04:37.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:36.026522 23475 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:37.228641 23474 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1205 milliseconds
08:04:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:36.025406 23473 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.228679 23466 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:37.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.233412 23466 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:37.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.235805 23466 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.237174 23466 hybrid_clock.cc:648] HybridClock initialized: now 1779091477237156 us; error 51 us; skew 500 ppm
08:04:37.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.238050 23466 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:37.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.245224 23466 webserver.cc:533] Webserver started at http://127.20.164.66:35829/ using document root <none> and password file <none>
08:04:37.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.246145 23466 fs_manager.cc:362] Metadata directory not provided
08:04:37.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.246389 23466 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:37.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.255546 23466 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.001s
08:04:37.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.260569 23487 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:04:37.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.261715 23466 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
08:04:37.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.261996 23466 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:37.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e269425272f249e496b7c12b18f63076"
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:47 on dist-test-slave-0rvw"
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.263881 23466 fs_report.cc:389] FS layout report
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/wal
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/data
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:37.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.327961 23466 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:04:37.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.329710 23466 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:37.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.330103 23466 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:37.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.333261 23466 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:37.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.340526 23494 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
08:04:37.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.349020 23466 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:04:37.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.349259 23466 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.003s
08:04:37.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.349606 23466 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:04:37.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.354307 23466 ts_tablet_manager.cc:616] Registered 1 tablets
08:04:37.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.354547 23466 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
08:04:37.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.354918 23494 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Bootstrap starting.
08:04:37.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.450762 23494 log.cc:826] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Log is configured to *not* fsync() on all Append() calls
08:04:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.600960 23494 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.602698 23494 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Bootstrap complete.
08:04:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.604790 23494 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Time spent bootstrapping tablet: real 0.250s	user 0.156s	sys 0.066s
08:04:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.622602 23466 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.66:39351
08:04:37.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.622763 23602 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.66:39351 every 8 connection(s)
08:04:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.625447 23466 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-1/data/info.pb
08:04:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.630090 23494 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:37.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.633498 23494 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: e269425272f249e496b7c12b18f63076, State: Initialized, Role: FOLLOWER
08:04:37.636 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:37.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.634692 23494 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:37.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.635562 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 23466
08:04:37.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.681443 23494 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076: Time spent starting tablet: real 0.076s	user 0.048s	sys 0.000s
08:04:37.685 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:04:37.691 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.164.65:46489
08:04:37.691 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:04:37.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.692837 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 23280
08:04:37.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.694140 23604 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:04:37.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.694633 23604 heartbeater.cc:461] Registering TS with master...
08:04:37.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.696054 23604 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:04:37.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.700505 21258 ts_manager.cc:194] Re-registered known tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:37.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.703891 23603 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:04:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.704188 23603 heartbeater.cc:461] Registering TS with master...
08:04:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.704954 23603 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:04:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.705129 23611 heartbeater.cc:344] Connected to a master server at 127.20.164.126:37373
08:04:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.705430 23611 heartbeater.cc:461] Registering TS with master...
08:04:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.706218 23611 heartbeater.cc:507] Master 127.20.164.126:37373 requested a full tablet report, sending...
08:04:37.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.707940 21338 ts_manager.cc:194] Re-registered known tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:37.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.710232 21338 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.66:56383
08:04:37.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.710282 21182 ts_manager.cc:194] Re-registered known tserver with Master: e269425272f249e496b7c12b18f63076 (127.20.164.66:39351)
08:04:37.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:37.713413 23603 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:04:37.752 [DEBUG - kudu-nio-5] (Connection.java:239) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] handling channelInactive
08:04:37.752 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state READY due to: connection closed
08:04:37.753 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:37.754 [DEBUG - Time-limited test] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:37.754 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.755 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.755 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.776 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:37.776 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:37.777 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1749490025(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@2965ebca -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7db3e4 -> wakeup thread Time-limited test)) after lookup
08:04:37.777 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:37.779 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.779 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.779 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.795 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:37.795 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:37.796 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [26ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [26ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1749490025(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@2965ebca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@980833519 after retry RPC@1306735318), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7db3e4 -> wakeup thread Time-limited test -> (continuation of Deferred@980833519 after retry RPC@1306735318))) after lookup
08:04:37.796 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:37.797 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.797 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.797 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.836 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:37.836 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:37.837 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=83), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [26ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [26ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [43ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [44ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [45ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1749490025(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@2965ebca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@980833519 after retry RPC@1306735318) -> (continuation of Deferred@1934984133 after retry RPC@670333691), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7db3e4 -> wakeup thread Time-limited test -> (continuation of Deferred@980833519 after retry RPC@1306735318) -> (continuation of Deferred@1934984133 after retry RPC@670333691))) after lookup
08:04:37.837 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:37.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.838 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.937 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:37.937 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:37.937 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=184), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [26ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [26ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [43ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [44ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [45ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [84ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [85ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [85ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [180ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [184ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1749490025(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@2965ebca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@980833519 after retry RPC@1306735318) -> (continuation of Deferred@1934984133 after retry RPC@670333691) -> (continuation of Deferred@509713214 after retry RPC@1897883070), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7db3e4 -> wakeup thread Time-limited test -> (continuation of Deferred@980833519 after retry RPC@1306735318) -> (continuation of Deferred@1934984133 after retry RPC@670333691) -> (continuation of Deferred@509713214 after retry RPC@1897883070))) after lookup
08:04:37.938 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:37.939 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.940 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:37.940 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:38.276 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:38.276 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251),e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)[L]]}]
08:04:38.277 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=523), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [26ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [26ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [43ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [44ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [45ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [84ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [85ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [85ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [180ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [184ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [185ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [187ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [187ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [519ms] refreshing cache from master, [519ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [523ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1749490025(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@2965ebca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@980833519 after retry RPC@1306735318) -> (continuation of Deferred@1934984133 after retry RPC@670333691) -> (continuation of Deferred@509713214 after retry RPC@1897883070) -> (continuation of Deferred@1530116157 after retry RPC@1359680624), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7db3e4 -> wakeup thread Time-limited test -> (continuation of Deferred@980833519 after retry RPC@1306735318) -> (continuation of Deferred@1934984133 after retry RPC@670333691) -> (continuation of Deferred@509713214 after retry RPC@1897883070) -> (continuation of Deferred@1530116157 after retry RPC@1359680624))) after lookup
08:04:38.277 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] connecting to peer
08:04:38.278 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:38.278 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] cleaning up while in state CONNECTING due to: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:38.278 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489) for tablet 7ce36047551045458a0528924b6fb281: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489
08:04:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.751147 23615 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader e8aea6cb22324b45b188c416f3d49634)
08:04:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.751654 23615 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:38.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.753597 23615 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:38.772994 23114 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:04:38.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.779218 23557 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076" is_pre_election: true
08:04:38.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.780053 23557 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d31321473b59420090c29f4de9f6f928 in term 9.
08:04:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.781567 23112 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e269425272f249e496b7c12b18f63076; no voters: e8aea6cb22324b45b188c416f3d49634
08:04:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.782377 23615 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 10 FOLLOWER]: Leader pre-election won for term 11
08:04:38.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.782707 23615 raft_consensus.cc:493] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 10 FOLLOWER]: Starting leader election (detected failure of leader e8aea6cb22324b45b188c416f3d49634)
08:04:38.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.783032 23615 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 10 FOLLOWER]: Advancing to term 11
08:04:38.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.789602 23615 raft_consensus.cc:515] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:38.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.792187 23615 leader_election.cc:290] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 11 election: Requested vote from peers e269425272f249e496b7c12b18f63076 (127.20.164.66:39351), e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:38.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.792078 23557 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "7ce36047551045458a0528924b6fb281" candidate_uuid: "d31321473b59420090c29f4de9f6f928" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "e269425272f249e496b7c12b18f63076"
08:04:38.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.792598 23557 raft_consensus.cc:3060] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 9 FOLLOWER]: Advancing to term 11
08:04:38.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:38.796383 23114 leader_election.cc:336] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111)
08:04:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.802304 23557 raft_consensus.cc:2468] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d31321473b59420090c29f4de9f6f928 in term 11.
08:04:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.803452 23112 leader_election.cc:304] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d31321473b59420090c29f4de9f6f928, e269425272f249e496b7c12b18f63076; no voters: e8aea6cb22324b45b188c416f3d49634
08:04:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.804189 23615 raft_consensus.cc:2804] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 11 FOLLOWER]: Leader election won for term 11
08:04:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.805287 23615 raft_consensus.cc:697] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [term 11 LEADER]: Becoming Leader. State: Replica: d31321473b59420090c29f4de9f6f928, State: Running, Role: LEADER
08:04:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.806716 23615 consensus_queue.cc:237] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:38.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:38.816241 21338 catalog_manager.cc:5696] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 reported cstate change: term changed from 10 to 11, leader changed from e8aea6cb22324b45b188c416f3d49634 (127.20.164.65) to d31321473b59420090c29f4de9f6f928 (127.20.164.67). New cstate: current_term: 11 leader_uuid: "d31321473b59420090c29f4de9f6f928" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } health_report { overall_health: UNKNOWN } } }
08:04:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:39.314242 23557 raft_consensus.cc:1275] T 7ce36047551045458a0528924b6fb281 P e269425272f249e496b7c12b18f63076 [term 11 FOLLOWER]: Refusing update from remote peer d31321473b59420090c29f4de9f6f928: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
08:04:39.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:39.315493 23633 consensus_queue.cc:1048] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
08:04:39.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:39.354940 23114 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 -> Peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Couldn't send request to peer e8aea6cb22324b45b188c416f3d49634. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:39.517 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ce36047551045458a0528924b6fb281 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318' with partition [<start>, <end>)
08:04:39.517 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=7ce36047551045458a0528924b6fb281@[d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)[L],e269425272f249e496b7c12b18f63076(127.20.164.66:39351),e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)]}]
08:04:39.518 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=7ce36047551045458a0528924b6fb281, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1764), Traces: [0ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [2ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [2ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [23ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [24ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [26ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [26ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [42ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [43ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [44ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [45ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [84ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [85ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [85ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [180ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [184ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [185ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [187ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [187ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [519ms] refreshing cache from master, [519ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [523ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, [524ms] sending RPC to server e8aea6cb22324b45b188c416f3d49634, [525ms] received response from server e8aea6cb22324b45b188c416f3d49634: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [526ms] delaying RPC due to: Network error: Failed to connect to peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489): Connection refused: /127.20.164.65:46489, [1760ms] refreshing cache from master, [1760ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.164.124:42449, [1764ms] Sub RPC GetTableLocations: received response from server master-127.20.164.124:42449: OK, deferred=Deferred@1749490025(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fd330336958@2965ebca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@980833519 after retry RPC@1306735318) -> (continuation of Deferred@1934984133 after retry RPC@670333691) -> (continuation of Deferred@509713214 after retry RPC@1897883070) -> (continuation of Deferred@1530116157 after retry RPC@1359680624) -> (continuation of Deferred@1722124576 after retry RPC@1862580496), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a7db3e4 -> wakeup thread Time-limited test -> (continuation of Deferred@980833519 after retry RPC@1306735318) -> (continuation of Deferred@1934984133 after retry RPC@670333691) -> (continuation of Deferred@509713214 after retry RPC@1897883070) -> (continuation of Deferred@1530116157 after retry RPC@1359680624) -> (continuation of Deferred@1722124576 after retry RPC@1862580496))) after lookup
08:04:39.518 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] connecting to peer
08:04:39.519 [DEBUG - kudu-nio-4] (Connection.java:757) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] Successfully connected to peer
08:04:39.541 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0x80d2e3c7, L:/127.0.0.1:56030 - R:/127.20.164.67:37251] using TOKEN/null
08:04:39.580 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.164.65:46489
08:04:39.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:39.582001 21137 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/logs
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.65:46489
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.65
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33865
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.65
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:04:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:39.888039 23649 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:39.888480 23649 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:39.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:39.888891 23649 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:39.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:39.922668 23649 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:39.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:39.923456 23649 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.164.65
08:04:39.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:39.959558 23649 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.164.84:33243
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.164.65:46489
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.164.65
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33865
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.164.126:37373,127.20.164.125:40721,127.20.164.124:42449
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.164.65
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/logs
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:25 UTC on e7f111948823
08:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11955
08:04:39.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:39.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:39.960790 23649 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:39.962216 23649 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:39.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:39.973903 23657 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:40.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:40.306182 23114 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111) [suppressed 4 similar messages]
08:04:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:39.975688 23658 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:41.383387 23656 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 23649
08:04:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:41.634019 23649 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.659s	user 0.000s	sys 0.002s
08:04:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:41.634374 23649 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.659s	user 0.000s	sys 0.002s
08:04:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:41.635841 23660 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:41.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:41.638301 23659 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1658 milliseconds
08:04:41.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.638389 23649 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:41.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.639810 23649 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:41.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.642436 23649 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:41.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.644003 23649 hybrid_clock.cc:648] HybridClock initialized: now 1779091481643955 us; error 36 us; skew 500 ppm
08:04:41.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.645119 23649 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:41.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.653036 23649 webserver.cc:533] Webserver started at http://127.20.164.65:33865/ using document root <none> and password file <none>
08:04:41.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.654281 23649 fs_manager.cc:362] Metadata directory not provided
08:04:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.654731 23649 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:41.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.664503 23649 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.001s	sys 0.003s
08:04:41.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.668715 23671 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:04:41.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.669804 23649 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.004s
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.670081 23649 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data,/tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e8aea6cb22324b45b188c416f3d49634"
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:44 on dist-test-slave-0rvw"
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.672045 23649 fs_report.cc:389] FS layout report
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/wal
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/data
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:41.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.735497 23649 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:04:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.737071 23649 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.737558 23649 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:41.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.740159 23649 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.745615 23678 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
08:04:41.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.753436 23649 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:04:41.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.753680 23649 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.001s	sys 0.001s
08:04:41.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.754019 23649 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:04:41.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.758606 23649 ts_tablet_manager.cc:616] Registered 1 tablets
08:04:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.758824 23649 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
08:04:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.759137 23678 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap starting.
08:04:41.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:41.822511 23114 consensus_peers.cc:597] T 7ce36047551045458a0528924b6fb281 P d31321473b59420090c29f4de9f6f928 -> Peer e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489): Couldn't send request to peer e8aea6cb22324b45b188c416f3d49634. Status: Network error: Client connection negotiation failed: client connection to 127.20.164.65:46489: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:04:41.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.841841 23678 log.cc:826] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Log is configured to *not* fsync() on all Append() calls
08:04:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.950145 23649 rpc_server.cc:307] RPC server started. Bound to: 127.20.164.65:46489
08:04:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.950372 23787 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.164.65:46489 every 8 connection(s)
08:04:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.952802 23649 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskjmZNRn/test-tmp/mini-kudu-cluster2975941203278269917/ts-0/data/info.pb
08:04:41.960 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:04:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.959342 21137 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu as pid 23649
08:04:41.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.982340 23678 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:41.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.983821 23678 tablet_bootstrap.cc:492] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Bootstrap complete.
08:04:41.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:41.985479 23678 ts_tablet_manager.cc:1403] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Time spent bootstrapping tablet: real 0.227s	user 0.141s	sys 0.080s
08:04:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.004407 23788 heartbeater.cc:344] Connected to a master server at 127.20.164.124:42449
08:04:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.005051 23788 heartbeater.cc:461] Registering TS with master...
08:04:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.003130 23678 raft_consensus.cc:359] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.006213 23678 raft_consensus.cc:740] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: e8aea6cb22324b45b188c416f3d49634, State: Initialized, Role: FOLLOWER
08:04:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.006536 23788 heartbeater.cc:507] Master 127.20.164.124:42449 requested a full tablet report, sending...
08:04:42.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.007244 23678 consensus_queue.cc:260] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d31321473b59420090c29f4de9f6f928" member_type: VOTER last_known_addr { host: "127.20.164.67" port: 37251 } } peers { permanent_uuid: "e269425272f249e496b7c12b18f63076" member_type: VOTER last_known_addr { host: "127.20.164.66" port: 39351 } } peers { permanent_uuid: "e8aea6cb22324b45b188c416f3d49634" member_type: VOTER last_known_addr { host: "127.20.164.65" port: 46489 } }
08:04:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.015296 23789 heartbeater.cc:344] Connected to a master server at 127.20.164.125:40721
08:04:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.015733 23789 heartbeater.cc:461] Registering TS with master...
08:04:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.016824 23789 heartbeater.cc:507] Master 127.20.164.125:40721 requested a full tablet report, sending...
08:04:42.028 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:04:42.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.029990 21258 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.034017 21337 ts_manager.cc:194] Re-registered known tserver with Master: e8aea6cb22324b45b188c416f3d49634 (127.20.164.65:46489)
08:04:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.036939 21337 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.164.65:53363
08:04:42.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.044551 23678 ts_tablet_manager.cc:1434] T 7ce36047551045458a0528924b6fb281 P e8aea6cb22324b45b188c416f3d49634: Time spent starting tablet: real 0.059s	user 0.029s	sys 0.015s
08:04:42.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.045012 23788 heartbeater.cc:499] Master 127.20.164.124:42449 was elected leader, sending a full tablet report...
08:04:42.055 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091430318, tablet=7ce36047551045458a0528924b6fb281, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:04:42.061 [DEBUG - main] (Connection.java:575) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] disconnecting while in state READY
08:04:42.062 [DEBUG - main] (Connection.java:575) [peer master-127.20.164.124:42449(127.20.164.124:42449)] disconnecting while in state READY
08:04:42.062 [DEBUG - main] (Connection.java:575) [peer master-127.20.164.125:40721(127.20.164.125:40721)] disconnecting while in state READY
08:04:42.063 [DEBUG - main] (Connection.java:575) [peer e269425272f249e496b7c12b18f63076(127.20.164.66:39351)] disconnecting while in state TERMINATED
08:04:42.063 [DEBUG - main] (Connection.java:575) [peer master-127.20.164.126:37373(127.20.164.126:37373)] disconnecting while in state READY
08:04:42.063 [DEBUG - main] (Connection.java:575) [peer e8aea6cb22324b45b188c416f3d49634(127.20.164.65:46489)] disconnecting while in state TERMINATED
08:04:42.064 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.164.124:42449(127.20.164.124:42449)] handling channelInactive
08:04:42.064 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.164.124:42449(127.20.164.124:42449)] cleaning up while in state READY due to: connection closed
08:04:42.065 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.164.126:37373(127.20.164.126:37373)] handling channelInactive
08:04:42.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.164.126:37373(127.20.164.126:37373)] cleaning up while in state READY due to: connection closed
08:04:42.065 [DEBUG - kudu-nio-4] (Connection.java:239) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] handling channelInactive
08:04:42.066 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d31321473b59420090c29f4de9f6f928(127.20.164.67:37251)] cleaning up while in state READY due to: connection closed
08:04:42.066 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:04:42.066 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.20.164.125:40721(127.20.164.125:40721)] handling channelInactive
08:04:42.066 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.20.164.125:40721(127.20.164.125:40721)] cleaning up while in state READY due to: connection closed
08:04:42.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.069836 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 23649
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:04:42.103695 23798 negotiation.cc:336] Failed RPC negotiation. Trace:
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.012638 (+     0us) reactor.cc:678] Submitting negotiation task for server connection from 127.20.164.65:59965 (local address 127.20.164.126:37373)
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.046203 (+ 33565us) server_negotiation.cc:207] Beginning negotiation
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.046231 (+    28us) server_negotiation.cc:400] Waiting for connection header
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.046280 (+    49us) server_negotiation.cc:408] Connection header received
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.050213 (+  3933us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.050244 (+    31us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.050478 (+   234us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.050674 (+   196us) server_negotiation.cc:227] Negotiated authn=SASL
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.059739 (+  9065us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.061160 (+  1421us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.079215 (+ 18055us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.079796 (+   581us) server_negotiation.cc:658] Negotiated auth-only TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.080098 (+   302us) server_negotiation.cc:366] Received SASL_INITIATE NegotiatePB request
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.080195 (+    97us) server_negotiation.cc:882] Received SASL_INITIATE request from client
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.080225 (+    30us) server_negotiation.cc:893] Client requested to use mechanism: PLAIN
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.080287 (+    62us) server_negotiation.cc:923] calling sasl_server_start()
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.080376 (+    89us) server_negotiation.cc:1092] Received PLAIN auth, user=slave
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.080426 (+    50us) server_negotiation.cc:378] Sending SASL_SUCCESS NegotiatePB response
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.080565 (+   139us) server_negotiation.cc:1036] Waiting for connection context
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 08:04:42.102050 (+ 21485us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.20.164.65:59965: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":12962,"thread_start_us":31867,"threads_started":1}
08:04:42.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.108560 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 23466
08:04:42.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.137420 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 23090
08:04:42.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.161474 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 21150
08:04:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.188470 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 21220
08:04:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:04:42.217813 21137 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskjmZNRn/build/tsan/bin/kudu with pid 21295
08:04:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T08:04:43Z chronyd exiting

Time: 71.006

OK (1 test)

