JUnit version 4.13.2
.08:03:20.482 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1779091400475
08:03:20.530 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
08:03:20.583 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:03:20.594 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrHGDhh/test-tmp
08:03:20.599 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
08:03:20.602 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrHGDhh/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
08:03:20.812 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

08:03:20.823 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
08:03:20.825 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

08:03:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.827448 17129 mini_oidc.cc:150] Starting JWKS server
08:03:20.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.829296 17129 webserver.cc:533] Webserver started at http://127.0.0.1:33617/ using document root <none> and password file <none>
08:03:20.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.829496 17129 mini_oidc.cc:186] Starting OIDC Discovery server
08:03:20.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.829784 17129 webserver.cc:533] Webserver started at http://127.0.0.1:33605/ using document root <none> and password file <none>
08:03:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T08:03:20Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:03:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T08:03:20Z Disabled control of system clock
08:03:20.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.843413 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:20.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:20.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/wal
08:03:20.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/data
08:03:20.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:20.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:20.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:20.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:20.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:20.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:20.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/logs
08:03:20.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/data/info.pb
08:03:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.126:40625
08:03:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.126
08:03:20.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:20.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:20.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:20.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:20.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:20.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:20.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:20.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:20.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33617/jwks.json with env {}
08:03:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.942070 17140 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:20.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.942265 17140 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:20.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.942320 17140 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.945850 17140 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.945905 17140 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.945931 17140 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:20.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.945957 17140 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:20.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.945979 17140 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:20.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.945998 17140 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33617/jwks.json
08:03:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.950085 17140 master_runner.cc:387] Master server non-default flags:
08:03:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/data
08:03:20.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/wal
08:03:20.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:20.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:20.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:20.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:20.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:20.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.126:40625
08:03:20.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33617/jwks.json
08:03:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:20.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/data/info.pb
08:03:20.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.126
08:03:20.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:20.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:20.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17140
08:03:20.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:20.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:20.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:20.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:20.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/logs
08:03:20.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:20.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:20.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:20.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.951361 17140 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.952734 17140 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:20.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.959092 17145 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:20.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.959100 17146 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:20.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:20.959391 17148 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:20.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.959971 17140 server_base.cc:1061] running on GCE node
08:03:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.960489 17140 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.961539 17140 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.962694 17140 hybrid_clock.cc:648] HybridClock initialized: now 1779091400962676 us; error 29 us; skew 500 ppm
08:03:20.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.964293 17140 webserver.cc:533] Webserver started at http://127.16.186.126:39083/ using document root <none> and password file <none>
08:03:20.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.964869 17140 fs_manager.cc:362] Metadata directory not provided
08:03:20.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.964926 17140 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:20.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.965129 17140 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:20.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.966704 17140 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/data/instance:
08:03:20.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c945442b9cb4691a8027dd09ea65855"
08:03:20.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:20 on dist-test-slave-lw51"
08:03:20.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.967175 17140 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/wal/instance:
08:03:20.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c945442b9cb4691a8027dd09ea65855"
08:03:20.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:20 on dist-test-slave-lw51"
08:03:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.970485 17140 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.004s	sys 0.000s
08:03:20.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.972404 17154 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:20.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.973276 17140 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:20.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.973376 17140 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/wal
08:03:20.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c945442b9cb4691a8027dd09ea65855"
08:03:20.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:20 on dist-test-slave-lw51"
08:03:20.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:20.973495 17140 fs_report.cc:389] FS layout report
08:03:20.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:20.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/wal
08:03:20.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/wal
08:03:20.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/data/data
08:03:20.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:20.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:20.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:20.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:20.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:21.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.005898 17140 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:21.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.006713 17140 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:21.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.006899 17140 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:21.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.013556 17140 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.126:40625
08:03:21.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.013568 17206 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.126:40625 every 8 connection(s)
08:03:21.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.014674 17140 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/data/info.pb
08:03:21.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.018112 17207 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.019081 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 17140
08:03:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.019214 17129 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-0/wal/instance
08:03:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.020637 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:21.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:21.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/wal
08:03:21.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/data
08:03:21.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:21.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:21.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/logs
08:03:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/data/info.pb
08:03:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.125:39845
08:03:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.125
08:03:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33617/jwks.json with env {}
08:03:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.022511 17207 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.035588 17207 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.036309 17157 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.125:39845: connect: Connection refused (error 111)
08:03:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.037146 17207 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.186.125:39845: Network error: Client connection negotiation failed: client connection to 127.16.186.125:39845: connect: Connection refused (error 111)
08:03:21.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.086345 17207 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } attempt: 1
08:03:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.087841 17207 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.186.125:39845: Network error: Client connection negotiation failed: client connection to 127.16.186.125:39845: connect: Connection refused (error 111)
08:03:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.119642 17209 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.119840 17209 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.119884 17209 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.123057 17209 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.123107 17209 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.123126 17209 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:21.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.123142 17209 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:21.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.123157 17209 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:21.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.123169 17209 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33617/jwks.json
08:03:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.126864 17209 master_runner.cc:387] Master server non-default flags:
08:03:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/data
08:03:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/wal
08:03:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.125:39845
08:03:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33617/jwks.json
08:03:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/data/info.pb
08:03:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.125
08:03:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17209
08:03:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/logs
08:03:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:21.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:21.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:21.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:21.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:21.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.127857 17209 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:21.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.128947 17209 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.134876 17217 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.135001 17218 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.135277 17209 server_base.cc:1061] running on GCE node
08:03:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.135169 17220 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.135855 17209 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:21.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.136890 17209 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:21.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.138064 17209 hybrid_clock.cc:648] HybridClock initialized: now 1779091401138048 us; error 31 us; skew 500 ppm
08:03:21.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.140111 17209 webserver.cc:533] Webserver started at http://127.16.186.125:37055/ using document root <none> and password file <none>
08:03:21.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.140761 17209 fs_manager.cc:362] Metadata directory not provided
08:03:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.140834 17209 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.141037 17209 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.142951 17209 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/data/instance:
08:03:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4fc0d468e404d9eb77dbb2a5c780e50"
08:03:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.143498 17209 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/wal/instance:
08:03:21.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4fc0d468e404d9eb77dbb2a5c780e50"
08:03:21.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.147300 17209 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.005s	sys 0.000s
08:03:21.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.149714 17226 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:21.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.150823 17209 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.001s
08:03:21.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.150930 17209 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/wal
08:03:21.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4fc0d468e404d9eb77dbb2a5c780e50"
08:03:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.151018 17209 fs_report.cc:389] FS layout report
08:03:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/wal
08:03:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/wal
08:03:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/data/data
08:03:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:21.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.156004 17207 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } attempt: 2
08:03:21.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.157346 17207 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.186.125:39845: Network error: Client connection negotiation failed: client connection to 127.16.186.125:39845: connect: Connection refused (error 111)
08:03:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.165114 17209 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.165732 17209 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.165877 17209 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:21.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.172281 17278 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.125:39845 every 8 connection(s)
08:03:21.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.172286 17209 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.125:39845
08:03:21.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.173252 17209 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/data/info.pb
08:03:21.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.176244 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 17209
08:03:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.176342 17129 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-1/wal/instance
08:03:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.176631 17279 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.177057 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/wal
08:03:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/data
08:03:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/logs
08:03:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/data/info.pb
08:03:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.124:44471
08:03:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.124
08:03:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33617/jwks.json with env {}
08:03:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.180744 17279 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.190146 17279 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.196596 17279 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.197335 17230 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.124:44471: connect: Connection refused (error 111)
08:03:21.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.201294 17279 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.186.124:44471: Network error: Client connection negotiation failed: client connection to 127.16.186.124:44471: connect: Connection refused (error 111)
08:03:21.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.248500 17207 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } attempt: 3
08:03:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.250495 17279 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } attempt: 1
08:03:21.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.252211 17279 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.186.124:44471: Network error: Client connection negotiation failed: client connection to 127.16.186.124:44471: connect: Connection refused (error 111)
08:03:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.252821 17207 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.253980 17207 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.186.124:44471: Network error: Client connection negotiation failed: client connection to 127.16.186.124:44471: connect: Connection refused (error 111)
08:03:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.279764 17281 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.279975 17281 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.280035 17281 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:21.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.283507 17281 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.283568 17281 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.283594 17281 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.283617 17281 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.283638 17281 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.283656 17281 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33617/jwks.json
08:03:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.285131 17207 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } attempt: 1
08:03:21.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.286417 17207 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.186.124:44471: Network error: Client connection negotiation failed: client connection to 127.16.186.124:44471: connect: Connection refused (error 111)
08:03:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.287854 17281 master_runner.cc:387] Master server non-default flags:
08:03:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/data
08:03:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/wal
08:03:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.124:44471
08:03:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:21.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33617/jwks.json
08:03:21.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/data/info.pb
08:03:21.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.124
08:03:21.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:21.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17281
08:03:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/logs
08:03:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:21.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:21.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:21.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:21.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:21.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:21.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:21.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:21.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.288869 17281 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:21.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.290079 17281 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:21.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.296123 17292 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.296123 17290 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.296514 17281 server_base.cc:1061] running on GCE node
08:03:21.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.296146 17289 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.297884 17281 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:21.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.298981 17281 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:21.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.300438 17281 hybrid_clock.cc:648] HybridClock initialized: now 1779091401300387 us; error 65 us; skew 500 ppm
08:03:21.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.302361 17281 webserver.cc:533] Webserver started at http://127.16.186.124:42339/ using document root <none> and password file <none>
08:03:21.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.302999 17281 fs_manager.cc:362] Metadata directory not provided
08:03:21.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.303076 17281 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:21.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.303287 17281 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:21.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.304893 17281 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/data/instance:
08:03:21.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "031d7cc38e01427ca373ed7f72e7660f"
08:03:21.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.305281 17281 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/wal/instance:
08:03:21.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "031d7cc38e01427ca373ed7f72e7660f"
08:03:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.308429 17281 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.002s	sys 0.000s
08:03:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.310320 17298 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:21.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.311350 17281 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:21.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.311480 17281 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/wal
08:03:21.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "031d7cc38e01427ca373ed7f72e7660f"
08:03:21.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.311568 17281 fs_report.cc:389] FS layout report
08:03:21.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/wal
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/wal
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/data/data
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.320384 17279 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } attempt: 2
08:03:21.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.321560 17279 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.186.124:44471: Network error: Client connection negotiation failed: client connection to 127.16.186.124:44471: connect: Connection refused (error 111)
08:03:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.335119 17281 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.335731 17281 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.335867 17281 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:21.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.342585 17350 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.124:44471 every 8 connection(s)
08:03:21.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.342598 17281 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.124:44471
08:03:21.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.343547 17281 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/data/info.pb
08:03:21.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.346149 17351 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:21.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.349294 17351 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.352633 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 17281
08:03:21.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.352728 17129 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/master-2/wal/instance
08:03:21.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.357223 17351 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.360924 17351 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.362637 17207 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } attempt: 2
08:03:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.368948 17351 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f: Bootstrap starting.
08:03:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.369704 17207 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855: Bootstrap starting.
08:03:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.371402 17351 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f: Neither blocks nor log segments found. Creating new log.
08:03:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.371896 17207 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855: Neither blocks nor log segments found. Creating new log.
08:03:21.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.372478 17351 log.cc:826] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f: Log is configured to *not* fsync() on all Append() calls
08:03:21.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.372853 17207 log.cc:826] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855: Log is configured to *not* fsync() on all Append() calls
08:03:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.374128 17351 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f: No bootstrap required, opened a new log
08:03:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.374356 17207 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855: No bootstrap required, opened a new log
08:03:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.377207 17207 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } }
08:03:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.377207 17351 raft_consensus.cc:359] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } }
08:03:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.377380 17207 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.377384 17351 raft_consensus.cc:385] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.377429 17207 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4c945442b9cb4691a8027dd09ea65855, State: Initialized, Role: FOLLOWER
08:03:21.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.377429 17351 raft_consensus.cc:740] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 031d7cc38e01427ca373ed7f72e7660f, State: Initialized, Role: FOLLOWER
08:03:21.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.377861 17207 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } }
08:03:21.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.377861 17351 consensus_queue.cc:260] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } }
08:03:21.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.378947 17351 sys_catalog.cc:565] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [sys.catalog]: configured and running, proceeding with master startup.
08:03:21.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.378947 17207 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [sys.catalog]: configured and running, proceeding with master startup.
08:03:21.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.379237 17362 sys_catalog.cc:455] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } } }
08:03:21.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.379966 17363 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } } }
08:03:21.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.380321 17363 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [sys.catalog]: This master's current role is: FOLLOWER
08:03:21.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.379478 17362 sys_catalog.cc:458] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [sys.catalog]: This master's current role is: FOLLOWER
08:03:21.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.384040 17385 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:21.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.384346 17385 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.383884 17382 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.386241 17382 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:21.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.386061 17140 master_runner.cc:187] Error getting master registration for 127.16.186.125:39845: OK, instance_id { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" instance_seqno: 1779091401170525 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:21.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.388093 17140 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:03:21.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.387539 17281 master_runner.cc:187] Error getting master registration for 127.16.186.125:39845: OK, instance_id { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" instance_seqno: 1779091401170525 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:21.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.389319 17281 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:03:21.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.412752 17279 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } attempt: 3
08:03:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.419773 17279 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50: Bootstrap starting.
08:03:21.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.422456 17279 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50: Neither blocks nor log segments found. Creating new log.
08:03:21.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.423539 17279 log.cc:826] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50: Log is configured to *not* fsync() on all Append() calls
08:03:21.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.425338 17279 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50: No bootstrap required, opened a new log
08:03:21.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.428504 17279 raft_consensus.cc:359] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } }
08:03:21.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.428691 17279 raft_consensus.cc:385] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.428745 17279 raft_consensus.cc:740] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c4fc0d468e404d9eb77dbb2a5c780e50, State: Initialized, Role: FOLLOWER
08:03:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.429168 17279 consensus_queue.cc:260] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } }
08:03:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.429739 17389 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } } }
08:03:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.430109 17389 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [sys.catalog]: This master's current role is: FOLLOWER
08:03:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.430403 17279 sys_catalog.cc:565] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [sys.catalog]: configured and running, proceeding with master startup.
08:03:21.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.433036 17363 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:21.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.433173 17363 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } }
08:03:21.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.434132 17363 leader_election.cc:290] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c4fc0d468e404d9eb77dbb2a5c780e50 (127.16.186.125:39845), 031d7cc38e01427ca373ed7f72e7660f (127.16.186.124:44471)
08:03:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.434501 17253 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c945442b9cb4691a8027dd09ea65855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" is_pre_election: true
08:03:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.434722 17253 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4c945442b9cb4691a8027dd09ea65855 in term 0.
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.435351 17157 leader_election.cc:304] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c945442b9cb4691a8027dd09ea65855, c4fc0d468e404d9eb77dbb2a5c780e50; no voters: 
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.435762 17325 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c945442b9cb4691a8027dd09ea65855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "031d7cc38e01427ca373ed7f72e7660f" is_pre_election: true
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.435973 17325 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4c945442b9cb4691a8027dd09ea65855 in term 0.
08:03:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.438062 17363 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.438134 17363 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.438174 17363 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 0 FOLLOWER]: Advancing to term 1
08:03:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.439069 17209 master_runner.cc:264] No leader master found from master c4fc0d468e404d9eb77dbb2a5c780e50
08:03:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.439121 17209 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.439198 17363 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } }
08:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.439538 17363 leader_election.cc:290] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [CANDIDATE]: Term 1 election: Requested vote from peers c4fc0d468e404d9eb77dbb2a5c780e50 (127.16.186.125:39845), 031d7cc38e01427ca373ed7f72e7660f (127.16.186.124:44471)
08:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.439810 17325 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c945442b9cb4691a8027dd09ea65855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "031d7cc38e01427ca373ed7f72e7660f"
08:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.439908 17325 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [term 0 FOLLOWER]: Advancing to term 1
08:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.439926 17253 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c945442b9cb4691a8027dd09ea65855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50"
08:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.440019 17253 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [term 0 FOLLOWER]: Advancing to term 1
08:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.440920 17325 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4c945442b9cb4691a8027dd09ea65855 in term 1.
08:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.441002 17253 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4c945442b9cb4691a8027dd09ea65855 in term 1.
08:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.441284 17158 leader_election.cc:304] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 031d7cc38e01427ca373ed7f72e7660f, 4c945442b9cb4691a8027dd09ea65855; no voters: 
08:03:21.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.442899 17363 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 1 FOLLOWER]: Leader election won for term 1
08:03:21.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.443084 17363 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [term 1 LEADER]: Becoming Leader. State: Replica: 4c945442b9cb4691a8027dd09ea65855, State: Running, Role: LEADER
08:03:21.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.443361 17363 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } }
08:03:21.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.445546 17363 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c945442b9cb4691a8027dd09ea65855. Latest consensus state: current_term: 1 leader_uuid: "4c945442b9cb4691a8027dd09ea65855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } } }
08:03:21.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.445645 17363 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [sys.catalog]: This master's current role is: LEADER
08:03:21.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.446339 17400 catalog_manager.cc:1593] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:21.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.446398 17400 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:21.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.446667 17404 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
08:03:21.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.447566 17404 catalog_manager.cc:1519] Initializing Kudu cluster ID...
08:03:21.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.468547 17325 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [term 1 FOLLOWER]: Refusing update from remote peer 4c945442b9cb4691a8027dd09ea65855: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:21.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.468547 17253 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [term 1 FOLLOWER]: Refusing update from remote peer 4c945442b9cb4691a8027dd09ea65855: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:21.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.469703 17402 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:21.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.470058 17402 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [LEADER]: Connected to new peer: Peer: permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:21.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.475003 17389 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c945442b9cb4691a8027dd09ea65855. Latest consensus state: current_term: 1 leader_uuid: "4c945442b9cb4691a8027dd09ea65855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } } }
08:03:21.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.475106 17389 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [sys.catalog]: This master's current role is: FOLLOWER
08:03:21.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.478515 17363 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4c945442b9cb4691a8027dd09ea65855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } } }
08:03:21.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.478616 17363 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [sys.catalog]: This master's current role is: LEADER
08:03:21.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.479835 17389 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4c945442b9cb4691a8027dd09ea65855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } } }
08:03:21.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.479923 17389 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50 [sys.catalog]: This master's current role is: FOLLOWER
08:03:21.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.481243 17405 mvcc.cc:204] Tried to move back new op lower bound from 7287158380408467456 to 7287158380317265920. Current Snapshot: MvccSnapshot[applied={T|T < 7287158380408467456}]
08:03:21.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.483326 17362 sys_catalog.cc:455] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c945442b9cb4691a8027dd09ea65855. Latest consensus state: current_term: 1 leader_uuid: "4c945442b9cb4691a8027dd09ea65855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } } }
08:03:21.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.483455 17362 sys_catalog.cc:458] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [sys.catalog]: This master's current role is: FOLLOWER
08:03:21.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.489909 17404 catalog_manager.cc:1382] Generated new cluster ID: d531a375f7e243969068430bfc98826e
08:03:21.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.489970 17404 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
08:03:21.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.509943 17406 mvcc.cc:204] Tried to move back new op lower bound from 7287158380408467456 to 7287158380317265920. Current Snapshot: MvccSnapshot[applied={T|T < 7287158380408467456}]
08:03:21.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.512460 17402 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4c945442b9cb4691a8027dd09ea65855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } } }
08:03:21.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.512593 17402 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855 [sys.catalog]: This master's current role is: LEADER
08:03:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.515533 17362 sys_catalog.cc:455] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4c945442b9cb4691a8027dd09ea65855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c945442b9cb4691a8027dd09ea65855" member_type: VOTER last_known_addr { host: "127.16.186.126" port: 40625 } } peers { permanent_uuid: "c4fc0d468e404d9eb77dbb2a5c780e50" member_type: VOTER last_known_addr { host: "127.16.186.125" port: 39845 } } peers { permanent_uuid: "031d7cc38e01427ca373ed7f72e7660f" member_type: VOTER last_known_addr { host: "127.16.186.124" port: 44471 } } }
08:03:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.515640 17362 sys_catalog.cc:458] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f [sys.catalog]: This master's current role is: FOLLOWER
08:03:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.533694 17404 catalog_manager.cc:1405] Generated new certificate authority record
08:03:21.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.535183 17404 catalog_manager.cc:1539] Loading token signing keys...
08:03:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.546841 17404 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 4c945442b9cb4691a8027dd09ea65855: Generated new TSK 0
08:03:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.547799 17404 catalog_manager.cc:1549] Initializing in-progress tserver states...
08:03:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.559401 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data
08:03:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/logs
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.65:0
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.65
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.65
08:03:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:21.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:21.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.672901 17413 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.673075 17413 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.673115 17413 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:21.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.676247 17413 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.676340 17413 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.65
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.679997 17413 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.65:0
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.65
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17413
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.65
08:03:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/logs
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.681238 17413 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.682631 17413 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.688511 17419 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.688501 17421 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.688810 17413 server_base.cc:1061] running on GCE node
08:03:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.689163 17418 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.689564 17413 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.690097 17413 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:21.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.691253 17413 hybrid_clock.cc:648] HybridClock initialized: now 1779091401691202 us; error 68 us; skew 500 ppm
08:03:21.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.692886 17413 webserver.cc:533] Webserver started at http://127.16.186.65:39271/ using document root <none> and password file <none>
08:03:21.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.693415 17413 fs_manager.cc:362] Metadata directory not provided
08:03:21.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.693503 17413 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:21.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.693725 17413 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.695133 17413 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/instance:
08:03:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13ed5f47b5c14458a6cc3efb8918cd2c"
08:03:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.695879 17413 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal/instance:
08:03:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13ed5f47b5c14458a6cc3efb8918cd2c"
08:03:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.698948 17413 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.003s	sys 0.000s
08:03:21.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.700858 17427 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.702013 17413 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.702111 17413 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13ed5f47b5c14458a6cc3efb8918cd2c"
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.702195 17413 fs_report.cc:389] FS layout report
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/data
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.722203 17413 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:21.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.722939 17413 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:21.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.723114 17413 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:21.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.723724 17413 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.724733 17413 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.724779 17413 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
08:03:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.724830 17413 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.724849 17413 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:03:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.733915 17413 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.65:40953
08:03:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.733956 17540 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.65:40953 every 8 connection(s)
08:03:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.734789 17413 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:21.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.738297 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 17413
08:03:21.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.738400 17129 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal/instance
08:03:21.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.740482 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:21.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:21.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:21.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data
08:03:21.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:21.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/logs
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.66:0
08:03:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.66
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.66
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.743841 17543 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.744019 17543 heartbeater.cc:461] Registering TS with master...
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.744416 17543 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.744782 17542 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.744860 17542 heartbeater.cc:461] Registering TS with master...
08:03:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.745033 17542 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.745708 17243 ts_manager.cc:194] Registered new tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.745954 17170 ts_manager.cc:194] Registered new tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.746066 17541 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.746129 17541 heartbeater.cc:461] Registering TS with master...
08:03:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.746279 17541 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:21.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.747210 17315 ts_manager.cc:194] Registered new tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:21.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.747323 17170 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.65:41841
08:03:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.840445 17548 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.840629 17548 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.840682 17548 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.843991 17548 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.844107 17548 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.66
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.847846 17548 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.66:0
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.66
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17548
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.66
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/logs
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.849159 17548 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:21.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.850390 17548 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:21.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.856346 17554 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.856370 17556 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.856447 17548 server_base.cc:1061] running on GCE node
08:03:21.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:21.856519 17553 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.856829 17548 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:21.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.857270 17548 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:21.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.858404 17548 hybrid_clock.cc:648] HybridClock initialized: now 1779091401858392 us; error 28 us; skew 500 ppm
08:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.859970 17548 webserver.cc:533] Webserver started at http://127.16.186.66:37919/ using document root <none> and password file <none>
08:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.860553 17548 fs_manager.cc:362] Metadata directory not provided
08:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.860608 17548 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.860826 17548 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.862277 17548 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/instance:
08:03:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.862772 17548 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal/instance:
08:03:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:21.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.865926 17548 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.003s	sys 0.000s
08:03:21.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.867815 17562 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.868669 17548 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.868767 17548 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.868853 17548 fs_report.cc:389] FS layout report
08:03:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/data
08:03:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:21.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.887279 17548 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:21.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.887975 17548 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:21.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.888109 17548 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:21.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.888567 17548 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:21.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.889534 17548 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:21.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.889580 17548 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
08:03:21.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.889616 17548 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:21.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.889629 17548 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:03:21.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.897660 17548 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.66:46109
08:03:21.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.897715 17675 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.66:46109 every 8 connection(s)
08:03:21.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.898648 17548 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:21.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.899273 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 17548
08:03:21.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.899360 17129 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal/instance
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.901914 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/logs
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:21.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.67:0
08:03:21.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.67
08:03:21.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.67
08:03:21.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:21.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:21.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.908921 17676 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.908974 17677 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.909152 17676 heartbeater.cc:461] Registering TS with master...
08:03:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.909152 17677 heartbeater.cc:461] Registering TS with master...
08:03:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.909165 17678 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.909782 17677 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.909943 17678 heartbeater.cc:461] Registering TS with master...
08:03:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.909781 17676 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.910909 17170 ts_manager.cc:194] Registered new tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.910960 17315 ts_manager.cc:194] Registered new tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.911161 17678 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.911599 17170 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.66:45823
08:03:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:21.911779 17243 ts_manager.cc:194] Registered new tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:22.001058 17683 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:22.001242 17683 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:22.001295 17683 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:22.004500 17683 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:22.004594 17683 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.67
08:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.008268 17683 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data
08:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.67:0
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.67
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17683
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.67
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/logs
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.009320 17683 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.010372 17683 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:22.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:22.016197 17688 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:22.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:22.016185 17689 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:22.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:22.016207 17691 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:22.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.016364 17683 server_base.cc:1061] running on GCE node
08:03:22.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.016975 17683 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:22.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.017540 17683 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:22.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.018697 17683 hybrid_clock.cc:648] HybridClock initialized: now 1779091402018681 us; error 31 us; skew 500 ppm
08:03:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.020534 17683 webserver.cc:533] Webserver started at http://127.16.186.67:39371/ using document root <none> and password file <none>
08:03:22.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.021091 17683 fs_manager.cc:362] Metadata directory not provided
08:03:22.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.021147 17683 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:22.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.021342 17683 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:22.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.023077 17683 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/instance:
08:03:22.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010d2db890fc482bb61ce7809297487b"
08:03:22.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:22 on dist-test-slave-lw51"
08:03:22.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.023577 17683 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal/instance:
08:03:22.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010d2db890fc482bb61ce7809297487b"
08:03:22.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:22 on dist-test-slave-lw51"
08:03:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.027096 17683 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.005s	sys 0.000s
08:03:22.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.029237 17697 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.030102 17683 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.030198 17683 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010d2db890fc482bb61ce7809297487b"
08:03:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:22 on dist-test-slave-lw51"
08:03:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.030282 17683 fs_report.cc:389] FS layout report
08:03:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/data
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:22.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.050779 17683 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.051415 17683 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.051581 17683 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.052111 17683 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:22.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.053066 17683 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:22.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.053112 17683 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
08:03:22.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.053164 17683 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:22.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.053182 17683 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:03:22.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.061368 17683 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.67:41083
08:03:22.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.061414 17810 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.67:41083 every 8 connection(s)
08:03:22.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.062325 17683 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.068387 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 17683
08:03:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.068498 17129 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal/instance
08:03:22.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.073046 17813 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:22.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.073267 17813 heartbeater.cc:461] Registering TS with master...
08:03:22.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.073484 17812 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:22.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.073556 17812 heartbeater.cc:461] Registering TS with master...
08:03:22.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.073868 17812 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:22.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.073868 17813 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:22.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.074517 17811 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:22.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.074582 17811 heartbeater.cc:461] Registering TS with master...
08:03:22.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.074759 17811 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:22.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.074826 17170 ts_manager.cc:194] Registered new tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:22.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.075405 17243 ts_manager.cc:194] Registered new tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:22.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.075415 17315 ts_manager.cc:194] Registered new tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:22.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.075418 17170 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.67:37229
08:03:22.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.088204 17129 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:03:22.089 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
08:03:22.090 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

08:03:22.098 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.16.186.126"
      port: 40625
    }
    bound_http_address {
      host: "127.16.186.126"
      port: 39083
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.16.186.125"
      port: 39845
    }
    bound_http_address {
      host: "127.16.186.125"
      port: 37055
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.16.186.124"
      port: 44471
    }
    bound_http_address {
      host: "127.16.186.124"
      port: 42339
    }
  }
}

08:03:22.101 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

08:03:22.103 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.16.186.65"
      port: 40953
    }
    bound_http_address {
      host: "127.16.186.65"
      port: 39271
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.16.186.66"
      port: 46109
    }
    bound_http_address {
      host: "127.16.186.66"
      port: 37919
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.16.186.67"
      port: 41083
    }
    bound_http_address {
      host: "127.16.186.67"
      port: 39371
    }
  }
}

08:03:22.103 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:03:22.285 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
08:03:22.348 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
08:03:22.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.386046 17385 catalog_manager.cc:1294] Loaded cluster ID: d531a375f7e243969068430bfc98826e
08:03:22.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.386132 17385 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f: loading cluster ID for follower catalog manager: success
08:03:22.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.388451 17385 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f: acquiring CA information for follower catalog manager: success
08:03:22.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.389803 17385 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 031d7cc38e01427ca373ed7f72e7660f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.447726 17400 catalog_manager.cc:1294] Loaded cluster ID: d531a375f7e243969068430bfc98826e
08:03:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.447803 17400 catalog_manager.cc:1587] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50: loading cluster ID for follower catalog manager: success
08:03:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.450241 17400 catalog_manager.cc:1609] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50: acquiring CA information for follower catalog manager: success
08:03:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.451594 17400 catalog_manager.cc:1637] T 00000000000000000000000000000000 P c4fc0d468e404d9eb77dbb2a5c780e50: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:22.499 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.186.126:40625(127.16.186.126:40625)] connecting to peer
08:03:22.544 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.186.124:44471(127.16.186.124:44471)] connecting to peer
08:03:22.545 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.186.125:39845(127.16.186.125:39845)] connecting to peer
08:03:22.563 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.186.126:40625(127.16.186.126:40625)] Successfully connected to peer
08:03:22.563 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.186.124:44471(127.16.186.124:44471)] Successfully connected to peer
08:03:22.563 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.16.186.125:39845(127.16.186.125:39845)] Successfully connected to peer
08:03:22.670 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.186.126
08:03:22.670 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.186.124
08:03:22.671 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.186.125
08:03:22.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.748904 17542 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:22.879 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x1f87a2e8, L:/127.0.0.1:37766 - R:/127.16.186.125:39845] using SASL/PLAIN
08:03:22.879 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc1ceaa99, L:/127.0.0.1:49146 - R:/127.16.186.124:44471] using SASL/PLAIN
08:03:22.879 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x149bc9e6, L:/127.0.0.1:42918 - R:/127.16.186.126:40625] using SASL/PLAIN
08:03:22.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.913089 17677 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:22.913 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:22.914 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=447), Traces: [0ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.186.126:40625, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.186.124:44471, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.186.125:39845, [422ms] Sub RPC ConnectToMaster: received response from server master-127.16.186.124:44471: OK, [422ms] Sub RPC ConnectToMaster: received response from server master-127.16.186.125:39845: OK, [422ms] Sub RPC ConnectToMaster: received response from server master-127.16.186.126:40625: OK, deferred=null) after lookup
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.955124 17170 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42918:
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455"
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
08:03:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
08:03:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:22.956560 17170 catalog_manager.cc:7058] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.980468 17745 tablet_service.cc:1511] Processing CreateTablet for tablet da718a1094e44290ad9bb1c8d3f5b8f0 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455 [id=bfac7b17856a4316888150aa6c089ba4]), partition=RANGE (key) PARTITION UNBOUNDED
08:03:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.983493 17610 tablet_service.cc:1511] Processing CreateTablet for tablet da718a1094e44290ad9bb1c8d3f5b8f0 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455 [id=bfac7b17856a4316888150aa6c089ba4]), partition=RANGE (key) PARTITION UNBOUNDED
08:03:22.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.984341 17610 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet da718a1094e44290ad9bb1c8d3f5b8f0. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.987803 17475 tablet_service.cc:1511] Processing CreateTablet for tablet da718a1094e44290ad9bb1c8d3f5b8f0 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455 [id=bfac7b17856a4316888150aa6c089ba4]), partition=RANGE (key) PARTITION UNBOUNDED
08:03:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.988503 17475 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet da718a1094e44290ad9bb1c8d3f5b8f0. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:22.992 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table bfac7b17856a4316888150aa6c089ba4
08:03:22.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.989607 17745 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet da718a1094e44290ad9bb1c8d3f5b8f0. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:22.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.993697 17840 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap starting.
08:03:22.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.995512 17840 tablet_bootstrap.cc:654] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Neither blocks nor log segments found. Creating new log.
08:03:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.996174 17840 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Log is configured to *not* fsync() on all Append() calls
08:03:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.997608 17840 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: No bootstrap required, opened a new log
08:03:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.997762 17842 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Bootstrap starting.
08:03:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:22.997778 17840 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Time spent bootstrapping tablet: real 0.004s	user 0.001s	sys 0.003s
08:03:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.001106 17842 tablet_bootstrap.cc:654] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Neither blocks nor log segments found. Creating new log.
08:03:23.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.001919 17842 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Log is configured to *not* fsync() on all Append() calls
08:03:23.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.002463 17843 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Bootstrap starting.
08:03:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.003504 17842 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: No bootstrap required, opened a new log
08:03:23.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.003691 17842 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Time spent bootstrapping tablet: real 0.006s	user 0.003s	sys 0.000s
08:03:23.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.004312 17840 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.004525 17840 raft_consensus.cc:385] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.004585 17840 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 13ed5f47b5c14458a6cc3efb8918cd2c, State: Initialized, Role: FOLLOWER
08:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.005030 17840 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.005849 17840 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.000s
08:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.006685 17843 tablet_bootstrap.cc:654] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Neither blocks nor log segments found. Creating new log.
08:03:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.007485 17843 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Log is configured to *not* fsync() on all Append() calls
08:03:23.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.007869 17842 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:23.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.008206 17842 raft_consensus.cc:385] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.009194 17843 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: No bootstrap required, opened a new log
08:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.009366 17843 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Time spent bootstrapping tablet: real 0.007s	user 0.000s	sys 0.005s
08:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.012257 17843 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.012454 17843 raft_consensus.cc:385] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.012507 17843 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 010d2db890fc482bb61ce7809297487b, State: Initialized, Role: FOLLOWER
08:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.012912 17843 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.013074 17842 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3782bdbeb06c497ca787a0b7b2e6c918, State: Initialized, Role: FOLLOWER
08:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.014344 17842 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.015159 17842 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Time spent starting tablet: real 0.011s	user 0.004s	sys 0.000s
08:03:23.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.018189 17843 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Time spent starting tablet: real 0.009s	user 0.004s	sys 0.000s
08:03:23.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.018678 17812 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:23.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.025872 17845 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:23.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.026289 17845 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:23.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.027721 17845 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:23.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.035194 17765 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b" is_pre_election: true
08:03:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.035455 17765 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 13ed5f47b5c14458a6cc3efb8918cd2c in term 0.
08:03:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.035907 17430 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c; no voters: 
08:03:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.036165 17845 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.036239 17845 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.036283 17845 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 0 FOLLOWER]: Advancing to term 1
08:03:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.036996 17630 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" is_pre_election: true
08:03:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.037220 17630 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 13ed5f47b5c14458a6cc3efb8918cd2c in term 0.
08:03:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.037354 17845 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:23.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.038017 17845 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 1 election: Requested vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.041607 17765 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b"
08:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.041741 17765 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 0 FOLLOWER]: Advancing to term 1
08:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.041755 17630 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.041869 17630 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 0 FOLLOWER]: Advancing to term 1
08:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.042773 17765 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 13ed5f47b5c14458a6cc3efb8918cd2c in term 1.
08:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.042891 17630 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 13ed5f47b5c14458a6cc3efb8918cd2c in term 1.
08:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.043033 17430 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c; no voters: 
08:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.043185 17845 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 1 FOLLOWER]: Leader election won for term 1
08:03:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.043381 17845 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 1 LEADER]: Becoming Leader. State: Replica: 13ed5f47b5c14458a6cc3efb8918cd2c, State: Running, Role: LEADER
08:03:23.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.043969 17845 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.047073 17170 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c reported cstate change: term changed from 0 to 1, leader changed from <none> to 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65). New cstate: current_term: 1 leader_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: HEALTHY } } }
08:03:23.056 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table bfac7b17856a4316888150aa6c089ba4
08:03:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:23.064396 17814 tablet.cc:2404] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:23.074 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:23.082 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299992, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:23.084 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [8ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=null) after lookup
08:03:23.105 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:23.114 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] Successfully connected to peer
08:03:23.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:23.150381 17679 tablet.cc:2404] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:23.164 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xf67ff439, L:/127.0.0.1:57748 - R:/127.16.186.65:40953] using TOKEN/null
08:03:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.172139 17630 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 1 FOLLOWER]: Refusing update from remote peer 13ed5f47b5c14458a6cc3efb8918cd2c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:23.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.173655 17765 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 1 FOLLOWER]: Refusing update from remote peer 13ed5f47b5c14458a6cc3efb8918cd2c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:23.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.174723 17851 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:23.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.175127 17851 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [LEADER]: Connected to new peer: Peer: permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:23.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.195423 17856 mvcc.cc:204] Tried to move back new op lower bound from 7287158387385434112 to 7287158386870652928. Current Snapshot: MvccSnapshot[applied={T|T < 7287158387385434112}]
08:03:23.282 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:23.284 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.186.65:40953
08:03:23.285 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:03:23.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:23.286094 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 17413
08:03:23.294 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:23.298 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] handling channelInactive
08:03:23.299 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state READY due to: connection closed
08:03:23.299 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: connection closed
08:03:23.312 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:23.312 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:23.313 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [4ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [17ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1124812212(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@11457d0b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a9eca85 -> wakeup thread Time-limited test)) after lookup
08:03:23.314 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:23.316 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.316 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.317 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.332 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:23.332 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:23.333 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [4ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [17ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [18ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [36ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1124812212(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@11457d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a9eca85 -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097))) after lookup
08:03:23.333 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:23.335 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.336 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.336 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:23.351 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:23.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [4ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [17ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [18ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [36ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [38ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [41ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [41ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [55ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1124812212(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@11457d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a9eca85 -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562))) after lookup
08:03:23.352 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:23.354 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.354 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.354 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.491 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:23.492 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:23.493 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=197), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [4ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [17ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [18ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [36ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [38ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [41ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [41ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [55ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [57ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [59ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [59ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [193ms] refreshing cache from master, [193ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [196ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1124812212(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@11457d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562) -> (continuation of Deferred@1204988555 after retry RPC@1897090093), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a9eca85 -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562) -> (continuation of Deferred@1204988555 after retry RPC@1897090093))) after lookup
08:03:23.494 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:23.495 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.495 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.495 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.811 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:23.811 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:23.811 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=516), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [4ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [17ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [18ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [36ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [38ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [41ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [41ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [55ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [57ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [59ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [59ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [193ms] refreshing cache from master, [193ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [196ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [198ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [200ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [200ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [513ms] refreshing cache from master, [513ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [515ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1124812212(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@11457d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562) -> (continuation of Deferred@1204988555 after retry RPC@1897090093) -> (continuation of Deferred@127883918 after retry RPC@1158418376), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a9eca85 -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562) -> (continuation of Deferred@1204988555 after retry RPC@1897090093) -> (continuation of Deferred@127883918 after retry RPC@1158418376))) after lookup
08:03:23.812 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:23.813 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.813 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:23.813 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:24.131 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:24.132 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:24.132 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=837), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [4ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [17ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [18ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [36ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [38ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [41ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [41ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [55ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [57ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [59ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [59ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [193ms] refreshing cache from master, [193ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [196ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [198ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [200ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [200ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [513ms] refreshing cache from master, [513ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [515ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [517ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [518ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [518ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [833ms] refreshing cache from master, [833ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [836ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1124812212(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@11457d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562) -> (continuation of Deferred@1204988555 after retry RPC@1897090093) -> (continuation of Deferred@127883918 after retry RPC@1158418376) -> (continuation of Deferred@813689431 after retry RPC@631228709), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a9eca85 -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562) -> (continuation of Deferred@1204988555 after retry RPC@1897090093) -> (continuation of Deferred@127883918 after retry RPC@1158418376) -> (continuation of Deferred@813689431 after retry RPC@631228709))) after lookup
08:03:24.133 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:24.134 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:24.134 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:24.134 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.734892 17869 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 13ed5f47b5c14458a6cc3efb8918cd2c)
08:03:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.734948 17870 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 13ed5f47b5c14458a6cc3efb8918cd2c)
08:03:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.735018 17869 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.735059 17870 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:24.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.736145 17869 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.737246 17870 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:24.737289 17701 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:24.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:24.739172 17701 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:24.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:24.739591 17566 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:24.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:24.741514 17566 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.741847 17765 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b" is_pre_election: true
08:03:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.741890 17630 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "010d2db890fc482bb61ce7809297487b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" is_pre_election: true
08:03:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.741989 17630 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 010d2db890fc482bb61ce7809297487b in term 1.
08:03:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.741978 17765 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3782bdbeb06c497ca787a0b7b2e6c918 in term 1.
08:03:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.742283 17699 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.742283 17565 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.742640 17870 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:03:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.742690 17870 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 13ed5f47b5c14458a6cc3efb8918cd2c)
08:03:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.742719 17870 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 1 FOLLOWER]: Advancing to term 2
08:03:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.743436 17870 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.743729 17870 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 2 election: Requested vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.743884 17869 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 1 FOLLOWER]: Leader pre-election won for term 2
08:03:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.743934 17869 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 1 FOLLOWER]: Starting leader election (detected failure of leader 13ed5f47b5c14458a6cc3efb8918cd2c)
08:03:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.743971 17869 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 1 FOLLOWER]: Advancing to term 2
08:03:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.744014 17765 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b"
08:03:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.744875 17869 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:24.745040 17566 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.745244 17869 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 2 election: Requested vote from peers 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.745402 17765 raft_consensus.cc:2393] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 3782bdbeb06c497ca787a0b7b2e6c918 in current term 2: Already voted for candidate 010d2db890fc482bb61ce7809297487b in this term.
08:03:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.745813 17630 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "010d2db890fc482bb61ce7809297487b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.745957 17630 raft_consensus.cc:2393] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 010d2db890fc482bb61ce7809297487b in current term 2: Already voted for candidate 3782bdbeb06c497ca787a0b7b2e6c918 in this term.
08:03:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.746018 17565 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:24.746217 17701 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.746392 17699 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 010d2db890fc482bb61ce7809297487b; no voters: 13ed5f47b5c14458a6cc3efb8918cd2c, 3782bdbeb06c497ca787a0b7b2e6c918
08:03:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.746411 17870 raft_consensus.cc:2749] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
08:03:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:24.746562 17869 raft_consensus.cc:2749] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
08:03:25.911 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:25.912 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:25.912 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2617), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [4ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [17ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [18ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [36ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [38ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [41ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [41ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [55ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [57ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [59ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [59ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [193ms] refreshing cache from master, [193ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [196ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [198ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [200ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [200ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [513ms] refreshing cache from master, [513ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [515ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [517ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [518ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [518ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [833ms] refreshing cache from master, [833ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [836ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [838ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [839ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [840ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [2613ms] refreshing cache from master, [2613ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [2616ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1124812212(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@11457d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562) -> (continuation of Deferred@1204988555 after retry RPC@1897090093) -> (continuation of Deferred@127883918 after retry RPC@1158418376) -> (continuation of Deferred@813689431 after retry RPC@631228709) -> (continuation of Deferred@1989395665 after retry RPC@757001671), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a9eca85 -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562) -> (continuation of Deferred@1204988555 after retry RPC@1897090093) -> (continuation of Deferred@127883918 after retry RPC@1158418376) -> (continuation of Deferred@813689431 after retry RPC@631228709) -> (continuation of Deferred@1989395665 after retry RPC@757001671))) after lookup
08:03:25.913 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:25.914 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:25.914 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:25.915 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:26.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.673553 17879 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:26.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.673676 17879 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:26.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.674052 17879 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:26.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.674567 17765 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b" is_pre_election: true
08:03:26.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.674715 17765 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3782bdbeb06c497ca787a0b7b2e6c918 in term 2.
08:03:26.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.675063 17565 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 
08:03:26.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.675239 17879 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 2 FOLLOWER]: Leader pre-election won for term 3
08:03:26.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.675308 17879 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:26.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.675349 17879 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 2 FOLLOWER]: Advancing to term 3
08:03:26.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.675395 17566 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:26.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.676275 17879 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:26.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.676610 17879 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 3 election: Requested vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.676894 17765 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b"
08:03:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.677033 17765 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 2 FOLLOWER]: Advancing to term 3
08:03:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.678030 17765 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3782bdbeb06c497ca787a0b7b2e6c918 in term 3.
08:03:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.678319 17565 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 
08:03:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.678555 17566 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.678473 17879 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 3 FOLLOWER]: Leader election won for term 3
08:03:26.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.678817 17879 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 3 LEADER]: Becoming Leader. State: Replica: 3782bdbeb06c497ca787a0b7b2e6c918, State: Running, Role: LEADER
08:03:26.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.679133 17879 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.681707 17170 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 reported cstate change: term changed from 1 to 3, leader changed from 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65) to 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66). New cstate: current_term: 3 leader_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: UNKNOWN } } }
08:03:26.711 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:26.712 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:26.713 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=16, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=3417), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [4ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [17ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [18ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [36ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [38ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [41ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [41ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [55ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [57ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [59ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [59ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [193ms] refreshing cache from master, [193ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [196ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [198ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [200ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [200ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [513ms] refreshing cache from master, [513ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [515ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [517ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [518ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [518ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [833ms] refreshing cache from master, [833ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [836ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [838ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [839ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [840ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [2613ms] refreshing cache from master, [2613ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [2616ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [2618ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [2619ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [2620ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [3413ms] refreshing cache from master, [3414ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [3416ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1124812212(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@11457d0b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562) -> (continuation of Deferred@1204988555 after retry RPC@1897090093) -> (continuation of Deferred@127883918 after retry RPC@1158418376) -> (continuation of Deferred@813689431 after retry RPC@631228709) -> (continuation of Deferred@1989395665 after retry RPC@757001671) -> (continuation of Deferred@2077532725 after retry RPC@570300509), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a9eca85 -> wakeup thread Time-limited test -> (continuation of Deferred@1565750339 after retry RPC@1730575097) -> (continuation of Deferred@1940876268 after retry RPC@1395040562) -> (continuation of Deferred@1204988555 after retry RPC@1897090093) -> (continuation of Deferred@127883918 after retry RPC@1158418376) -> (continuation of Deferred@813689431 after retry RPC@631228709) -> (continuation of Deferred@1989395665 after retry RPC@757001671) -> (continuation of Deferred@2077532725 after retry RPC@570300509))) after lookup
08:03:26.714 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:26.715 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] Successfully connected to peer
08:03:26.733 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x59fe130b, L:/127.0.0.1:47100 - R:/127.16.186.66:46109] using TOKEN/null
08:03:26.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.737165 17765 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 3 FOLLOWER]: Refusing update from remote peer 3782bdbeb06c497ca787a0b7b2e6c918: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
08:03:26.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.737704 17882 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [LEADER]: Connected to new peer: Peer: permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
08:03:26.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.738111 17566 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 -> Peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Couldn't send request to peer 13ed5f47b5c14458a6cc3efb8918cd2c. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.741017 17897 mvcc.cc:204] Tried to move back new op lower bound from 7287158401990148096 to 7287158401759952896. Current Snapshot: MvccSnapshot[applied={T|T < 7287158401990148096}]
08:03:26.759 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.186.65:40953
08:03:26.760 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.761178 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data
08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/logs
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.65:40953
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.65
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39271
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.65
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:26.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.857098 17904 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:26.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.857278 17904 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:26.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.857319 17904 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:26.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.860499 17904 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:26.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.860595 17904 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.65
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.864205 17904 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.65:40953
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.65
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39271
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17904
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.65
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/logs
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.865145 17904 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:26.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.866199 17904 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:26.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.871999 17909 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:26.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.872000 17912 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:26.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:26.872063 17910 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:26.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.872179 17904 server_base.cc:1061] running on GCE node
08:03:26.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.872529 17904 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:26.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.873040 17904 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.874177 17904 hybrid_clock.cc:648] HybridClock initialized: now 1779091406874156 us; error 31 us; skew 500 ppm
08:03:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.875761 17904 webserver.cc:533] Webserver started at http://127.16.186.65:39271/ using document root <none> and password file <none>
08:03:26.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.876260 17904 fs_manager.cc:362] Metadata directory not provided
08:03:26.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.876331 17904 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:26.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.879305 17904 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
08:03:26.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.881042 17918 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.881903 17904 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.881986 17904 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13ed5f47b5c14458a6cc3efb8918cd2c"
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.882305 17904 fs_report.cc:389] FS layout report
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/data
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:26.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.900774 17904 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:26.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.901358 17904 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:26.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.901535 17904 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:26.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.902061 17904 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:26.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.903339 17925 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
08:03:26.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.906641 17904 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:26.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.906694 17904 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
08:03:26.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.906759 17904 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:26.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.908186 17904 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:26.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.908238 17904 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
08:03:26.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.908561 17925 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap starting.
08:03:26.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.919021 17904 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.65:40953
08:03:26.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.919062 18032 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.65:40953 every 8 connection(s)
08:03:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.920258 17904 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.924913 17925 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Log is configured to *not* fsync() on all Append() calls
08:03:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.926276 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 17904
08:03:26.926 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:26.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.933521 17925 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:26.935 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:26.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.933832 18033 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:26.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.934046 18033 heartbeater.cc:461] Registering TS with master...
08:03:26.935 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.186.66:46109
08:03:26.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.934070 17925 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap complete.
08:03:26.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.934602 18033 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:26.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.935066 17925 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Time spent bootstrapping tablet: real 0.027s	user 0.016s	sys 0.009s
08:03:26.936 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:03:26.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.936569 17314 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.936751 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 17548
08:03:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.937198 17925 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.937758 17925 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 13ed5f47b5c14458a6cc3efb8918cd2c, State: Initialized, Role: FOLLOWER
08:03:26.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.938326 17925 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.939142 18034 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.939224 18034 heartbeater.cc:461] Registering TS with master...
08:03:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.939289 17925 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
08:03:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.939404 18034 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:26.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.940487 17171 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:26.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.940956 18035 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:26.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.941020 18035 heartbeater.cc:461] Registering TS with master...
08:03:26.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.941190 18035 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:26.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.941370 17171 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.65:41671
08:03:26.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.942119 17242 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:26.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:26.943288 18034 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:26.945 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] handling channelInactive
08:03:26.945 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:26.945 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state READY due to: connection closed
08:03:26.946 [DEBUG - Time-limited test] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:26.947 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:26.947 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:26.947 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:26.971 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:26.971 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:26.971 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [3ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1545039739(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d7aa854 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41c95341 -> wakeup thread Time-limited test)) after lookup
08:03:26.972 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:26.973 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:26.973 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:26.973 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:26.990 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:26.990 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:26.991 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [3ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [27ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [28ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [28ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [45ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1545039739(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d7aa854 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@424224772 after retry RPC@303113015), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41c95341 -> wakeup thread Time-limited test -> (continuation of Deferred@424224772 after retry RPC@303113015))) after lookup
08:03:26.991 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:26.992 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:26.992 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:26.993 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:27.031 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:27.032 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:27.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=86), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [3ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [27ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [28ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [28ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [45ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [46ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [48ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [48ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [86ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1545039739(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d7aa854 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@424224772 after retry RPC@303113015) -> (continuation of Deferred@2072174026 after retry RPC@1611747606), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41c95341 -> wakeup thread Time-limited test -> (continuation of Deferred@424224772 after retry RPC@303113015) -> (continuation of Deferred@2072174026 after retry RPC@1611747606))) after lookup
08:03:27.034 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:27.035 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:27.035 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:27.035 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:27.170 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:27.171 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:27.171 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=225), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [3ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [27ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [28ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [28ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [45ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [46ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [48ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [48ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [86ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [88ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [90ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [90ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [223ms] refreshing cache from master, [223ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [225ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1545039739(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d7aa854 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@424224772 after retry RPC@303113015) -> (continuation of Deferred@2072174026 after retry RPC@1611747606) -> (continuation of Deferred@1431963411 after retry RPC@416292948), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41c95341 -> wakeup thread Time-limited test -> (continuation of Deferred@424224772 after retry RPC@303113015) -> (continuation of Deferred@2072174026 after retry RPC@1611747606) -> (continuation of Deferred@1431963411 after retry RPC@416292948))) after lookup
08:03:27.172 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:27.173 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:27.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:27.173 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:27.431 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:27.431 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:27.432 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=486), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [3ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [27ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [28ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [28ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [45ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [46ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [48ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [48ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [86ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [88ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [90ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [90ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [223ms] refreshing cache from master, [223ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [225ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [227ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [228ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [228ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [483ms] refreshing cache from master, [483ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [485ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1545039739(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d7aa854 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@424224772 after retry RPC@303113015) -> (continuation of Deferred@2072174026 after retry RPC@1611747606) -> (continuation of Deferred@1431963411 after retry RPC@416292948) -> (continuation of Deferred@1666921205 after retry RPC@1887547270), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41c95341 -> wakeup thread Time-limited test -> (continuation of Deferred@424224772 after retry RPC@303113015) -> (continuation of Deferred@2072174026 after retry RPC@1611747606) -> (continuation of Deferred@1431963411 after retry RPC@416292948) -> (continuation of Deferred@1666921205 after retry RPC@1887547270))) after lookup
08:03:27.432 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:27.433 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:27.433 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:27.434 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.259184 18047 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3782bdbeb06c497ca787a0b7b2e6c918)
08:03:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.259316 18047 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.259723 18047 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:28.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.260910 17699 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109): Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111)
08:03:28.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.264467 17987 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "010d2db890fc482bb61ce7809297487b" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" is_pre_election: true
08:03:28.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.264768 17987 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 010d2db890fc482bb61ce7809297487b in term 1.
08:03:28.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.265522 17701 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c; no voters: 3782bdbeb06c497ca787a0b7b2e6c918
08:03:28.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.265691 18047 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 3 FOLLOWER]: Leader pre-election won for term 4
08:03:28.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.265743 18047 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 3 FOLLOWER]: Starting leader election (detected failure of leader 3782bdbeb06c497ca787a0b7b2e6c918)
08:03:28.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.265779 18047 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 3 FOLLOWER]: Advancing to term 4
08:03:28.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.266569 18047 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:28.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.266954 18047 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 4 election: Requested vote from peers 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:28.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.267356 17987 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "010d2db890fc482bb61ce7809297487b" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c"
08:03:28.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.267489 17987 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 1 FOLLOWER]: Advancing to term 4
08:03:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.267886 17699 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109): Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111)
08:03:28.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.269191 17987 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 010d2db890fc482bb61ce7809297487b in term 4.
08:03:28.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.269498 17701 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c; no voters: 3782bdbeb06c497ca787a0b7b2e6c918
08:03:28.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.269670 18047 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 4 FOLLOWER]: Leader election won for term 4
08:03:28.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.269886 18047 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 4 LEADER]: Becoming Leader. State: Replica: 010d2db890fc482bb61ce7809297487b, State: Running, Role: LEADER
08:03:28.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.270187 18047 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:28.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.272488 17171 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b reported cstate change: term changed from 3 to 4, leader changed from 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66) to 010d2db890fc482bb61ce7809297487b (127.16.186.67). New cstate: current_term: 4 leader_uuid: "010d2db890fc482bb61ce7809297487b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: UNKNOWN } } }
08:03:28.651 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:28.651 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:28.652 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1706), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [3ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [27ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [28ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [28ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [45ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [46ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [48ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [48ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [86ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [88ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [90ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [90ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [223ms] refreshing cache from master, [223ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [225ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [227ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [228ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [228ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [483ms] refreshing cache from master, [483ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [485ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [487ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [489ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [489ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [1703ms] refreshing cache from master, [1703ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [1706ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1545039739(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d7aa854 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@424224772 after retry RPC@303113015) -> (continuation of Deferred@2072174026 after retry RPC@1611747606) -> (continuation of Deferred@1431963411 after retry RPC@416292948) -> (continuation of Deferred@1666921205 after retry RPC@1887547270) -> (continuation of Deferred@159767657 after retry RPC@1946533696), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41c95341 -> wakeup thread Time-limited test -> (continuation of Deferred@424224772 after retry RPC@303113015) -> (continuation of Deferred@2072174026 after retry RPC@1611747606) -> (continuation of Deferred@1431963411 after retry RPC@416292948) -> (continuation of Deferred@1666921205 after retry RPC@1887547270) -> (continuation of Deferred@159767657 after retry RPC@1946533696))) after lookup
08:03:28.653 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:28.654 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] Successfully connected to peer
08:03:28.674 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x299bedb1, L:/127.0.0.1:38150 - R:/127.16.186.67:41083] using TOKEN/null
08:03:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.677630 17987 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 4 FOLLOWER]: Refusing update from remote peer 010d2db890fc482bb61ce7809297487b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 4 index: 10. (index mismatch)
08:03:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.677852 17699 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b -> Peer 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109): Couldn't send request to peer 3782bdbeb06c497ca787a0b7b2e6c918. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.678247 18052 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [LEADER]: Connected to new peer: Peer: permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
08:03:28.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.685189 18068 mvcc.cc:204] Tried to move back new op lower bound from 7287158409937420288 to 7287158408277200896. Current Snapshot: MvccSnapshot[applied={T|T < 7287158409937420288}]
08:03:28.697 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.186.66:46109
08:03:28.697 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.698236 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/logs
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.66:46109
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.66
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37919
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.66
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:28.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.797396 18075 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:28.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.797621 18075 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.797683 18075 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.800921 18075 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.801033 18075 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.66
08:03:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.804754 18075 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.66:46109
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.66
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37919
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18075
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.66
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/logs
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.806063 18075 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.807113 18075 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.813231 18080 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.813220 18083 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.813336 18075 server_base.cc:1061] running on GCE node
08:03:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:28.813221 18081 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.813759 18075 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.814325 18075 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:28.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.815479 18075 hybrid_clock.cc:648] HybridClock initialized: now 1779091408815453 us; error 42 us; skew 500 ppm
08:03:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.817044 18075 webserver.cc:533] Webserver started at http://127.16.186.66:37919/ using document root <none> and password file <none>
08:03:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.817548 18075 fs_manager.cc:362] Metadata directory not provided
08:03:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.817621 18075 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:28.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.820531 18075 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
08:03:28.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.822224 18089 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.823163 18075 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.823246 18075 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.823616 18075 fs_report.cc:389] FS layout report
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/data
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:28.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.861080 18075 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:28.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.861739 18075 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:28.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.861884 18075 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:28.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.862391 18075 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:28.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.863577 18096 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
08:03:28.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.866729 18075 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:28.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.866778 18075 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
08:03:28.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.866822 18075 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:28.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.868162 18075 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:28.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.868202 18075 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
08:03:28.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.868530 18096 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Bootstrap starting.
08:03:28.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.877769 18075 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.66:46109
08:03:28.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.877816 18203 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.66:46109 every 8 connection(s)
08:03:28.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.878705 18075 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.883383 18096 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Log is configured to *not* fsync() on all Append() calls
08:03:28.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.885154 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 18075
08:03:28.885 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:28.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.888037 18204 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:28.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.888267 18204 heartbeater.cc:461] Registering TS with master...
08:03:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.888842 18204 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.889169 18205 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.889243 18205 heartbeater.cc:461] Registering TS with master...
08:03:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.889410 18205 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.890712 17314 ts_manager.cc:194] Re-registered known tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.891426 17171 ts_manager.cc:194] Re-registered known tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.892300 17171 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.66:39883
08:03:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.892907 18206 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.892974 18206 heartbeater.cc:461] Registering TS with master...
08:03:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.893137 18206 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:28.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.893880 18096 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:28.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.893934 17242 ts_manager.cc:194] Re-registered known tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:28.895 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:28.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.894245 18096 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Bootstrap complete.
08:03:28.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.895355 18096 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Time spent bootstrapping tablet: real 0.027s	user 0.014s	sys 0.010s
08:03:28.896 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.186.67:41083
08:03:28.896 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:03:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.896651 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 17683
08:03:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.898178 18096 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.898554 18096 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3782bdbeb06c497ca787a0b7b2e6c918, State: Initialized, Role: FOLLOWER
08:03:28.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.898986 18096 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.899784 18096 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Time spent starting tablet: real 0.004s	user 0.005s	sys 0.001s
08:03:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:28.899809 18205 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:28.903 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:28.903 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] handling channelInactive
08:03:28.903 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state READY due to: connection closed
08:03:28.904 [DEBUG - Time-limited test] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:28.905 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.905 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.905 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.910 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:28.910 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:28.910 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=7), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [1ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [6ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@996822615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@62cc09c5 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72e783d8 -> wakeup thread Time-limited test)) after lookup
08:03:28.911 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:28.912 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.912 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.912 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.931 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:28.931 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:28.931 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [1ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [6ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [7ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [8ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [8ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [27ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@996822615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@62cc09c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72e783d8 -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572))) after lookup
08:03:28.932 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:28.933 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.933 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.933 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:28.952 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:28.952 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [1ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [6ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [7ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [8ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [8ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [27ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [28ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [29ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [47ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@996822615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@62cc09c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572) -> (continuation of Deferred@114321356 after retry RPC@53149843), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72e783d8 -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572) -> (continuation of Deferred@114321356 after retry RPC@53149843))) after lookup
08:03:28.952 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:28.953 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.953 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:28.954 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:29.071 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:29.071 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:29.071 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=168), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [1ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [6ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [7ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [8ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [8ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [27ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [28ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [29ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [47ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [48ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [50ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [50ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [166ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@996822615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@62cc09c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572) -> (continuation of Deferred@114321356 after retry RPC@53149843) -> (continuation of Deferred@2013819577 after retry RPC@595385011), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72e783d8 -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572) -> (continuation of Deferred@114321356 after retry RPC@53149843) -> (continuation of Deferred@2013819577 after retry RPC@595385011))) after lookup
08:03:29.072 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:29.073 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:29.073 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:29.073 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:29.111 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:29.112 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:29.112 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=208), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [1ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [6ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [7ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [8ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [8ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [27ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [28ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [29ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [47ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [48ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [50ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [50ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [166ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [168ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [169ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [169ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [204ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [207ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@996822615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@62cc09c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572) -> (continuation of Deferred@114321356 after retry RPC@53149843) -> (continuation of Deferred@2013819577 after retry RPC@595385011) -> (continuation of Deferred@354077757 after retry RPC@1358312815), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72e783d8 -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572) -> (continuation of Deferred@114321356 after retry RPC@53149843) -> (continuation of Deferred@2013819577 after retry RPC@595385011) -> (continuation of Deferred@354077757 after retry RPC@1358312815))) after lookup
08:03:29.113 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:29.114 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:29.114 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:29.114 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:29.271 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:29.271 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:29.272 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=368), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [1ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [6ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [7ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [8ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [8ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [27ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [28ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [29ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [47ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [48ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [50ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [50ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [166ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [168ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [169ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [169ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [204ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [207ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [208ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [210ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [210ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [364ms] refreshing cache from master, [364ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [367ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@996822615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@62cc09c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572) -> (continuation of Deferred@114321356 after retry RPC@53149843) -> (continuation of Deferred@2013819577 after retry RPC@595385011) -> (continuation of Deferred@354077757 after retry RPC@1358312815) -> (continuation of Deferred@1575837067 after retry RPC@1757386594), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72e783d8 -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572) -> (continuation of Deferred@114321356 after retry RPC@53149843) -> (continuation of Deferred@2013819577 after retry RPC@595385011) -> (continuation of Deferred@354077757 after retry RPC@1358312815) -> (continuation of Deferred@1575837067 after retry RPC@1757386594))) after lookup
08:03:29.272 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:29.273 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:29.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:29.273 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:30.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.068857 18217 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:30.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.069036 18217 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:30.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.070125 18217 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:30.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:30.070838 18092 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:30.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:30.071623 18092 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.073766 17987 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" is_pre_election: true
08:03:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.074216 18093 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.074430 18217 raft_consensus.cc:2749] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
08:03:30.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.196756 18222 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 010d2db890fc482bb61ce7809297487b)
08:03:30.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.196889 18222 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.197940 18222 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:30.198575 17921 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:30.199355 17921 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:30.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.202694 18158 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" is_pre_election: true
08:03:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.202900 18158 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 13ed5f47b5c14458a6cc3efb8918cd2c in term 3.
08:03:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.203487 17920 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 13ed5f47b5c14458a6cc3efb8918cd2c, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 010d2db890fc482bb61ce7809297487b
08:03:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.203722 18222 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 4 FOLLOWER]: Leader pre-election won for term 5
08:03:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.203783 18222 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 4 FOLLOWER]: Starting leader election (detected failure of leader 010d2db890fc482bb61ce7809297487b)
08:03:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.203823 18222 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 4 FOLLOWER]: Advancing to term 5
08:03:30.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.204599 18222 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:30.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.204933 18222 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 5 election: Requested vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:30.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.205271 18158 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:30.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.205390 18158 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 3 FOLLOWER]: Advancing to term 5
08:03:30.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:30.205735 17921 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:30.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.207118 18158 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 13ed5f47b5c14458a6cc3efb8918cd2c in term 5.
08:03:30.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.207407 17920 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 13ed5f47b5c14458a6cc3efb8918cd2c, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 010d2db890fc482bb61ce7809297487b
08:03:30.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.207603 18222 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 5 FOLLOWER]: Leader election won for term 5
08:03:30.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.207811 18222 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 5 LEADER]: Becoming Leader. State: Replica: 13ed5f47b5c14458a6cc3efb8918cd2c, State: Running, Role: LEADER
08:03:30.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.208137 18222 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.210287 17171 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c reported cstate change: term changed from 4 to 5, leader changed from 010d2db890fc482bb61ce7809297487b (127.16.186.67) to 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65). New cstate: current_term: 5 leader_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: HEALTHY } } }
08:03:30.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.751188 18158 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 5 FOLLOWER]: Refusing update from remote peer 13ed5f47b5c14458a6cc3efb8918cd2c: Log matching property violated. Preceding OpId in replica: term: 3 index: 8. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
08:03:30.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:30.751847 18240 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
08:03:30.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:30.761927 17921 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c -> Peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Couldn't send request to peer 010d2db890fc482bb61ce7809297487b. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:32.911 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:32.912 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:32.912 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=4008), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [1ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [6ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [7ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [8ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [8ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [27ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [28ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [29ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [47ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [48ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [50ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [50ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [166ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [168ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [169ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [169ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [204ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [207ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [208ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [210ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [210ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [364ms] refreshing cache from master, [364ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [367ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [368ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [369ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [370ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [4004ms] refreshing cache from master, [4004ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [4007ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@996822615(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@62cc09c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572) -> (continuation of Deferred@114321356 after retry RPC@53149843) -> (continuation of Deferred@2013819577 after retry RPC@595385011) -> (continuation of Deferred@354077757 after retry RPC@1358312815) -> (continuation of Deferred@1575837067 after retry RPC@1757386594) -> (continuation of Deferred@1592136827 after retry RPC@1257675842), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72e783d8 -> wakeup thread Time-limited test -> (continuation of Deferred@1485166599 after retry RPC@372402572) -> (continuation of Deferred@114321356 after retry RPC@53149843) -> (continuation of Deferred@2013819577 after retry RPC@595385011) -> (continuation of Deferred@354077757 after retry RPC@1358312815) -> (continuation of Deferred@1575837067 after retry RPC@1757386594) -> (continuation of Deferred@1592136827 after retry RPC@1257675842))) after lookup
08:03:32.913 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:32.914 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] Successfully connected to peer
08:03:32.931 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xff9e6451, L:/127.0.0.1:50124 - R:/127.16.186.65:40953] using TOKEN/null
08:03:32.947 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.186.67:41083
08:03:32.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:03:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:32.948338 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data
08:03:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/logs
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.67:41083
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.67
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39371
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.67
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:33.044962 18267 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:33.045149 18267 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:33.045190 18267 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:33.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:33.048444 18267 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:33.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:33.048563 18267 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.67
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.052323 18267 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.67:41083
08:03:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.67
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39371
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18267
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.67
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/logs
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.053669 18267 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:33.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.054759 18267 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:33.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:33.060712 18273 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:33.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.060781 18267 server_base.cc:1061] running on GCE node
08:03:33.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:33.060726 18272 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:33.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:33.060799 18275 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:33.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.061775 18267 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:33.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.062248 18267 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:33.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.063400 18267 hybrid_clock.cc:648] HybridClock initialized: now 1779091413063389 us; error 25 us; skew 500 ppm
08:03:33.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.065025 18267 webserver.cc:533] Webserver started at http://127.16.186.67:39371/ using document root <none> and password file <none>
08:03:33.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.065532 18267 fs_manager.cc:362] Metadata directory not provided
08:03:33.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.065608 18267 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:33.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.068555 18267 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
08:03:33.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.070256 18281 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.071177 18267 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.071264 18267 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010d2db890fc482bb61ce7809297487b"
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:22 on dist-test-slave-lw51"
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.071633 18267 fs_report.cc:389] FS layout report
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/data
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.088573 18267 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:33.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.089099 18267 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:33.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.089231 18267 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:33.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.089762 18267 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.090858 18288 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
08:03:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.093859 18267 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:33.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.093921 18267 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
08:03:33.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.093966 18267 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:33.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.095327 18267 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:33.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.095373 18267 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
08:03:33.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.095690 18288 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Bootstrap starting.
08:03:33.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.104770 18267 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.67:41083
08:03:33.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.104802 18395 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.67:41083 every 8 connection(s)
08:03:33.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.105751 18267 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.111627 18288 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Log is configured to *not* fsync() on all Append() calls
08:03:33.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.113993 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 18267
08:03:33.114 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:33.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.116182 18398 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:33.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.116415 18398 heartbeater.cc:461] Registering TS with master...
08:03:33.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.117015 18398 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:33.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.117870 18396 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:33.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.117949 18396 heartbeater.cc:461] Registering TS with master...
08:03:33.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.118120 18396 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:33.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.118525 18397 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:33.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.118602 18397 heartbeater.cc:461] Registering TS with master...
08:03:33.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.118762 18397 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:33.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.118978 17314 ts_manager.cc:194] Re-registered known tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:33.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.119421 17243 ts_manager.cc:194] Re-registered known tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:33.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.120326 17170 ts_manager.cc:194] Re-registered known tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:33.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.121218 17170 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.67:48185
08:03:33.123 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:33.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.124939 18288 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:33.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.125350 18288 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Bootstrap complete.
08:03:33.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.126601 18288 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Time spent bootstrapping tablet: real 0.031s	user 0.025s	sys 0.004s
08:03:33.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.129392 18288 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:33.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.129824 18288 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 010d2db890fc482bb61ce7809297487b, State: Initialized, Role: FOLLOWER
08:03:33.130 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.186.65:40953
08:03:33.130 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:03:33.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.130295 18288 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:33.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.131098 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 17904
08:03:33.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.131222 18288 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Time spent starting tablet: real 0.005s	user 0.005s	sys 0.000s
08:03:33.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:33.132820 18397 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:33.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:33.139 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] handling channelInactive
08:03:33.139 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state READY due to: connection closed
08:03:33.149 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:33.150 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.150 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.150 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.171 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:33.172 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:33.173 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [11ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [11ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1937766186(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d95718b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60bb6364 -> wakeup thread Time-limited test)) after lookup
08:03:33.173 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:33.174 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.175 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.175 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.191 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:33.191 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:33.191 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=52), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [11ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [11ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [34ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [36ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [36ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1937766186(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d95718b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60bb6364 -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673))) after lookup
08:03:33.192 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:33.193 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.193 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.193 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.231 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:33.231 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:33.231 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=92), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [11ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [11ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [34ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [36ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [36ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [53ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [54ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [55ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [91ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1937766186(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d95718b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673) -> (continuation of Deferred@184517360 after retry RPC@1954804692), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60bb6364 -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673) -> (continuation of Deferred@184517360 after retry RPC@1954804692))) after lookup
08:03:33.232 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:33.233 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.233 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.234 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:33.351 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:33.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=212), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [11ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [11ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [34ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [36ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [36ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [53ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [54ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [55ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [91ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [93ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [94ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [95ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [211ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1937766186(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d95718b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673) -> (continuation of Deferred@184517360 after retry RPC@1954804692) -> (continuation of Deferred@173197552 after retry RPC@1629783869), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60bb6364 -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673) -> (continuation of Deferred@184517360 after retry RPC@1954804692) -> (continuation of Deferred@173197552 after retry RPC@1629783869))) after lookup
08:03:33.352 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:33.353 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.353 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.353 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.711 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:33.712 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:33.712 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=573), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [11ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [11ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [34ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [36ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [36ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [53ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [54ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [55ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [91ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [93ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [94ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [95ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [211ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [213ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [214ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [214ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [569ms] refreshing cache from master, [570ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [572ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1937766186(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d95718b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673) -> (continuation of Deferred@184517360 after retry RPC@1954804692) -> (continuation of Deferred@173197552 after retry RPC@1629783869) -> (continuation of Deferred@1696493947 after retry RPC@541466506), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60bb6364 -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673) -> (continuation of Deferred@184517360 after retry RPC@1954804692) -> (continuation of Deferred@173197552 after retry RPC@1629783869) -> (continuation of Deferred@1696493947 after retry RPC@541466506))) after lookup
08:03:33.713 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:33.714 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.714 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:33.714 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:34.311 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:34.312 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:34.312 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=13, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1173), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [11ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [11ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [34ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [36ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [36ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [53ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [54ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [55ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [91ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [93ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [94ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [95ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [211ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [213ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [214ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [214ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [569ms] refreshing cache from master, [570ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [572ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [573ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [575ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [575ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1170ms] refreshing cache from master, [1170ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [1172ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1937766186(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d95718b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673) -> (continuation of Deferred@184517360 after retry RPC@1954804692) -> (continuation of Deferred@173197552 after retry RPC@1629783869) -> (continuation of Deferred@1696493947 after retry RPC@541466506) -> (continuation of Deferred@485827210 after retry RPC@1345197103), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60bb6364 -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673) -> (continuation of Deferred@184517360 after retry RPC@1954804692) -> (continuation of Deferred@173197552 after retry RPC@1629783869) -> (continuation of Deferred@1696493947 after retry RPC@541466506) -> (continuation of Deferred@485827210 after retry RPC@1345197103))) after lookup
08:03:34.313 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:34.314 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:34.314 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:34.314 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:34.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.447053 18409 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 13ed5f47b5c14458a6cc3efb8918cd2c)
08:03:34.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.447188 18409 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:34.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.447573 18409 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:34.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:34.448797 18093 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:34.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.452123 18350 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b" is_pre_election: true
08:03:34.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.452382 18350 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3782bdbeb06c497ca787a0b7b2e6c918 in term 4.
08:03:34.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.453016 18092 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:34.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.453194 18409 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 5 FOLLOWER]: Leader pre-election won for term 6
08:03:34.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.453266 18409 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 13ed5f47b5c14458a6cc3efb8918cd2c)
08:03:34.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.453310 18409 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 5 FOLLOWER]: Advancing to term 6
08:03:34.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.454205 18409 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:34.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.454566 18409 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 6 election: Requested vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.454891 18350 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b"
08:03:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.455024 18350 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 4 FOLLOWER]: Advancing to term 6
08:03:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:34.455446 18093 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:34.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.456650 18350 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3782bdbeb06c497ca787a0b7b2e6c918 in term 6.
08:03:34.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.456936 18092 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:34.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.457082 18409 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 6 FOLLOWER]: Leader election won for term 6
08:03:34.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.457260 18409 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 6 LEADER]: Becoming Leader. State: Replica: 3782bdbeb06c497ca787a0b7b2e6c918, State: Running, Role: LEADER
08:03:34.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.457553 18409 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:34.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.459605 17170 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 reported cstate change: term changed from 5 to 6, leader changed from 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65) to 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66). New cstate: current_term: 6 leader_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: UNKNOWN } } }
08:03:34.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.895128 18350 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 6 FOLLOWER]: Refusing update from remote peer 3782bdbeb06c497ca787a0b7b2e6c918: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 6 index: 17. (index mismatch)
08:03:34.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:34.895978 18409 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [LEADER]: Connected to new peer: Peer: permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
08:03:34.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:34.904603 18093 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 -> Peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Couldn't send request to peer 13ed5f47b5c14458a6cc3efb8918cd2c. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:35.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:35.503410 18093 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111) [suppressed 3 similar messages]
08:03:37.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:37.418609 18093 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 -> Peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Couldn't send request to peer 13ed5f47b5c14458a6cc3efb8918cd2c. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:03:38.051 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:38.051 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:38.052 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=15, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=4913), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [11ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [11ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [34ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [36ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [36ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [53ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [54ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [55ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [89ms] refreshing cache from master, [89ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [91ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [93ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [94ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [95ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [211ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [213ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [214ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [214ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [569ms] refreshing cache from master, [570ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [572ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [573ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [575ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [575ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1170ms] refreshing cache from master, [1170ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [1172ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [1174ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [1175ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1175ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [4909ms] refreshing cache from master, [4909ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [4912ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1937766186(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d95718b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673) -> (continuation of Deferred@184517360 after retry RPC@1954804692) -> (continuation of Deferred@173197552 after retry RPC@1629783869) -> (continuation of Deferred@1696493947 after retry RPC@541466506) -> (continuation of Deferred@485827210 after retry RPC@1345197103) -> (continuation of Deferred@578342198 after retry RPC@47655482), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60bb6364 -> wakeup thread Time-limited test -> (continuation of Deferred@1959647015 after retry RPC@1339435673) -> (continuation of Deferred@184517360 after retry RPC@1954804692) -> (continuation of Deferred@173197552 after retry RPC@1629783869) -> (continuation of Deferred@1696493947 after retry RPC@541466506) -> (continuation of Deferred@485827210 after retry RPC@1345197103) -> (continuation of Deferred@578342198 after retry RPC@47655482))) after lookup
08:03:38.052 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:38.053 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] Successfully connected to peer
08:03:38.071 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x8c3b8ec6, L:/127.0.0.1:36856 - R:/127.16.186.66:46109] using TOKEN/null
08:03:38.089 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.186.65:40953
08:03:38.089 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:03:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.090094 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data
08:03:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/logs
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.65:40953
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.65
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39271
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.65
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.191049 18448 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.191251 18448 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.191293 18448 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:38.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.194509 18448 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:38.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.194607 18448 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.65
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.199504 18448 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.65:40953
08:03:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.65
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39271
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18448
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.65
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/logs
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.200524 18448 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.201591 18448 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:38.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.207736 18448 server_base.cc:1061] running on GCE node
08:03:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.207648 18455 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.207633 18454 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:38.207626 18457 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.208137 18448 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.208576 18448 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:38.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.209729 18448 hybrid_clock.cc:648] HybridClock initialized: now 1779091418209712 us; error 28 us; skew 500 ppm
08:03:38.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.211339 18448 webserver.cc:533] Webserver started at http://127.16.186.65:39271/ using document root <none> and password file <none>
08:03:38.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.211870 18448 fs_manager.cc:362] Metadata directory not provided
08:03:38.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.211928 18448 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.214985 18448 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
08:03:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.216647 18463 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.217602 18448 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.217685 18448 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13ed5f47b5c14458a6cc3efb8918cd2c"
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.218029 18448 fs_report.cc:389] FS layout report
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/data
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.228372 18448 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:38.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.228915 18448 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:38.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.229048 18448 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:38.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.229516 18448 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:38.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.230580 18470 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
08:03:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.233748 18448 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.233803 18448 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
08:03:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.233846 18448 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:38.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.235293 18448 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:38.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.235335 18448 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
08:03:38.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.235659 18470 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap starting.
08:03:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.244995 18448 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.65:40953
08:03:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.245038 18577 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.65:40953 every 8 connection(s)
08:03:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.246049 18448 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:38.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.250232 18470 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Log is configured to *not* fsync() on all Append() calls
08:03:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.255095 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 18448
08:03:38.255 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.259256 18578 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.259501 18578 heartbeater.cc:461] Registering TS with master...
08:03:38.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.260057 18578 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:38.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.262036 17313 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:38.263 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:38.263 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.186.66:46109
08:03:38.264 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.263854 18580 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.263962 18580 heartbeater.cc:461] Registering TS with master...
08:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.264173 18580 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.265076 17243 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.265353 18579 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.265360 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 18075
08:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.265424 18579 heartbeater.cc:461] Registering TS with master...
08:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.265622 18579 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.266433 18470 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.266834 18470 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap complete.
08:03:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.267590 17171 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.267923 18470 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Time spent bootstrapping tablet: real 0.033s	user 0.019s	sys 0.009s
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.268492 17171 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.65:48327
08:03:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.270224 18470 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.270532 18470 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 13ed5f47b5c14458a6cc3efb8918cd2c, State: Initialized, Role: FOLLOWER
08:03:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.270915 18470 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:38.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.271876 18470 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Time spent starting tablet: real 0.004s	user 0.005s	sys 0.000s
08:03:38.272 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] handling channelInactive
08:03:38.272 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state READY due to: connection closed
08:03:38.272 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:38.273 [DEBUG - Time-limited test] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:38.274 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.274 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.274 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:38.273962 18579 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:38.290 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:38.291 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:38.291 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [2ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1140368489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@78c5d0d6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f3d8bb -> wakeup thread Time-limited test)) after lookup
08:03:38.292 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:38.293 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.293 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.293 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.311 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:38.311 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:38.311 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [2ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [19ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [21ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [21ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1140368489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@78c5d0d6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@521794458 after retry RPC@129488578), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f3d8bb -> wakeup thread Time-limited test -> (continuation of Deferred@521794458 after retry RPC@129488578))) after lookup
08:03:38.312 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:38.313 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.313 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.313 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.330 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:38.331 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:38.331 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=58), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [2ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [19ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [21ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [21ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [40ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [41ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [41ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1140368489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@78c5d0d6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@521794458 after retry RPC@129488578) -> (continuation of Deferred@941854153 after retry RPC@26955209), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f3d8bb -> wakeup thread Time-limited test -> (continuation of Deferred@521794458 after retry RPC@129488578) -> (continuation of Deferred@941854153 after retry RPC@26955209))) after lookup
08:03:38.332 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:38.333 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.333 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.333 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.371 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:38.371 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:38.372 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=99), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [2ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [19ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [21ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [21ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [40ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [41ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [41ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [59ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [61ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [61ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [96ms] refreshing cache from master, [97ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [99ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1140368489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@78c5d0d6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@521794458 after retry RPC@129488578) -> (continuation of Deferred@941854153 after retry RPC@26955209) -> (continuation of Deferred@1359203983 after retry RPC@179119203), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f3d8bb -> wakeup thread Time-limited test -> (continuation of Deferred@521794458 after retry RPC@129488578) -> (continuation of Deferred@941854153 after retry RPC@26955209) -> (continuation of Deferred@1359203983 after retry RPC@179119203))) after lookup
08:03:38.372 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:38.373 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.373 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.374 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.571 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:38.571 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:38.572 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=299), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [2ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [19ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [21ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [21ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [40ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [41ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [41ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [59ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [61ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [61ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [96ms] refreshing cache from master, [97ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [99ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [100ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [102ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [102ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [296ms] refreshing cache from master, [296ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [299ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1140368489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@78c5d0d6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@521794458 after retry RPC@129488578) -> (continuation of Deferred@941854153 after retry RPC@26955209) -> (continuation of Deferred@1359203983 after retry RPC@179119203) -> (continuation of Deferred@2021748606 after retry RPC@468505137), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f3d8bb -> wakeup thread Time-limited test -> (continuation of Deferred@521794458 after retry RPC@129488578) -> (continuation of Deferred@941854153 after retry RPC@26955209) -> (continuation of Deferred@1359203983 after retry RPC@179119203) -> (continuation of Deferred@2021748606 after retry RPC@468505137))) after lookup
08:03:38.573 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:38.574 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.574 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:38.575 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:39.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.589480 18591 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 3782bdbeb06c497ca787a0b7b2e6c918)
08:03:39.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.589623 18591 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.590718 18591 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.591320 18283 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111)
08:03:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.592053 18283 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109): Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111)
08:03:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.595144 18532 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "010d2db890fc482bb61ce7809297487b" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" is_pre_election: true
08:03:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.595374 18532 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 010d2db890fc482bb61ce7809297487b in term 5.
08:03:39.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.595983 18285 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c; no voters: 3782bdbeb06c497ca787a0b7b2e6c918
08:03:39.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.596186 18591 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 6 FOLLOWER]: Leader pre-election won for term 7
08:03:39.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.596240 18591 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 6 FOLLOWER]: Starting leader election (detected failure of leader 3782bdbeb06c497ca787a0b7b2e6c918)
08:03:39.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.596267 18591 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 6 FOLLOWER]: Advancing to term 7
08:03:39.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.597000 18591 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:39.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.597280 18591 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 7 election: Requested vote from peers 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:39.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.597709 18532 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "010d2db890fc482bb61ce7809297487b" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c"
08:03:39.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.597838 18532 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 5 FOLLOWER]: Advancing to term 7
08:03:39.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:39.598366 18283 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109): Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111)
08:03:39.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.599448 18532 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 010d2db890fc482bb61ce7809297487b in term 7.
08:03:39.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.599727 18285 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c; no voters: 3782bdbeb06c497ca787a0b7b2e6c918
08:03:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.599880 18591 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 7 FOLLOWER]: Leader election won for term 7
08:03:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.600111 18591 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 7 LEADER]: Becoming Leader. State: Replica: 010d2db890fc482bb61ce7809297487b, State: Running, Role: LEADER
08:03:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.600561 18591 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:39.602852 17171 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b reported cstate change: term changed from 6 to 7, leader changed from 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66) to 010d2db890fc482bb61ce7809297487b (127.16.186.67). New cstate: current_term: 7 leader_uuid: "010d2db890fc482bb61ce7809297487b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: UNKNOWN } } }
08:03:40.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.017176 18283 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b -> Peer 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109): Couldn't send request to peer 3782bdbeb06c497ca787a0b7b2e6c918. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:40.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.155898 18532 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 7 FOLLOWER]: Refusing update from remote peer 010d2db890fc482bb61ce7809297487b: Log matching property violated. Preceding OpId in replica: term: 5 index: 16. Preceding OpId from leader: term: 7 index: 21. (index mismatch)
08:03:40.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.156597 18591 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [LEADER]: Connected to new peer: Peer: permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
08:03:40.171 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:40.171 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:40.172 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1899), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [2ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [2ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [19ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [21ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [21ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [40ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [41ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [41ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [59ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [61ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [61ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [96ms] refreshing cache from master, [97ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [99ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [100ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [102ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [102ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [296ms] refreshing cache from master, [296ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [299ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [301ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [302ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [303ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [1896ms] refreshing cache from master, [1896ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [1898ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1140368489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@78c5d0d6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@521794458 after retry RPC@129488578) -> (continuation of Deferred@941854153 after retry RPC@26955209) -> (continuation of Deferred@1359203983 after retry RPC@179119203) -> (continuation of Deferred@2021748606 after retry RPC@468505137) -> (continuation of Deferred@482547162 after retry RPC@518957402), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f3d8bb -> wakeup thread Time-limited test -> (continuation of Deferred@521794458 after retry RPC@129488578) -> (continuation of Deferred@941854153 after retry RPC@26955209) -> (continuation of Deferred@1359203983 after retry RPC@179119203) -> (continuation of Deferred@2021748606 after retry RPC@468505137) -> (continuation of Deferred@482547162 after retry RPC@518957402))) after lookup
08:03:40.173 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:40.174 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] Successfully connected to peer
08:03:40.192 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd90edeeb, L:/127.0.0.1:58542 - R:/127.16.186.67:41083] using TOKEN/null
08:03:40.209 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.186.66:46109
08:03:40.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:03:40.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.211644 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:40.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:40.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:40.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data
08:03:40.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:40.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:40.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/logs
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.66:46109
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.66
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37919
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.66
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:40.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.316599 18620 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:40.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.316818 18620 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:40.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.316860 18620 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:40.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.320103 18620 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:40.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.320209 18620 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.66
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.324033 18620 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.66:46109
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.66
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37919
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18620
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.66
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/logs
08:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.325100 18620 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.326202 18620 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.332214 18629 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.332222 18626 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.332667 18620 server_base.cc:1061] running on GCE node
08:03:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:40.332800 18627 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.333122 18620 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.333616 18620 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:40.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.334826 18620 hybrid_clock.cc:648] HybridClock initialized: now 1779091420334778 us; error 29 us; skew 500 ppm
08:03:40.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.336584 18620 webserver.cc:533] Webserver started at http://127.16.186.66:37919/ using document root <none> and password file <none>
08:03:40.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.337170 18620 fs_manager.cc:362] Metadata directory not provided
08:03:40.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.337229 18620 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:40.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.340392 18620 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
08:03:40.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.344508 18635 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:40.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.345707 18620 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.345805 18620 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.346243 18620 fs_report.cc:389] FS layout report
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/data
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:40.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.370537 18620 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.371130 18620 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.371263 18620 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.371743 18620 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.372876 18642 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
08:03:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.375993 18620 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.376051 18620 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
08:03:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.376102 18620 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:40.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.377451 18620 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:40.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.377529 18620 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.002s	sys 0.000s
08:03:40.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.377849 18642 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Bootstrap starting.
08:03:40.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.387480 18620 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.66:46109
08:03:40.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.387501 18749 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.66:46109 every 8 connection(s)
08:03:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.388744 18620 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:40.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.389259 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 18620
08:03:40.390 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:40.399 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:40.400 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.186.67:41083
08:03:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.399924 18642 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Log is configured to *not* fsync() on all Append() calls
08:03:40.400 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:03:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.400753 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 18267
08:03:40.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.403756 18751 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:40.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.404008 18751 heartbeater.cc:461] Registering TS with master...
08:03:40.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.404595 18751 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:40.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.406436 18752 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:40.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.406536 18752 heartbeater.cc:461] Registering TS with master...
08:03:40.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.406734 18752 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.406876 17170 ts_manager.cc:194] Re-registered known tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.407763 17243 ts_manager.cc:194] Re-registered known tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.408739 18750 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.408732 17170 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.66:59331
08:03:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.408824 18750 heartbeater.cc:461] Registering TS with master...
08:03:40.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.409022 18750 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:40.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.409896 17313 ts_manager.cc:194] Re-registered known tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:40.410 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] handling channelInactive
08:03:40.410 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state READY due to: connection closed
08:03:40.410 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:40.411 [DEBUG - Time-limited test] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:40.412 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.413 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.413 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.418154 18642 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:40.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.418465 18642 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Bootstrap complete.
08:03:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.419260 18642 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Time spent bootstrapping tablet: real 0.042s	user 0.019s	sys 0.012s
08:03:40.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.421209 18642 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:40.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.421561 18642 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3782bdbeb06c497ca787a0b7b2e6c918, State: Initialized, Role: FOLLOWER
08:03:40.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.421944 18642 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:40.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.422601 18751 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:40.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:40.422755 18642 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Time spent starting tablet: real 0.003s	user 0.005s	sys 0.000s
08:03:40.430 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:40.431 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:40.431 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [2ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1718304909(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d248163 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@481a1ff -> wakeup thread Time-limited test)) after lookup
08:03:40.432 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:40.432 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.433 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.433 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.450 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:40.451 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:40.451 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [2ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [20ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [22ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [22ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1718304909(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d248163 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1686439836 after retry RPC@1837292714), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@481a1ff -> wakeup thread Time-limited test -> (continuation of Deferred@1686439836 after retry RPC@1837292714))) after lookup
08:03:40.452 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:40.452 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.453 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.453 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.491 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:40.491 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:40.492 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=80), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [2ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [20ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [22ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [22ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [40ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [42ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [42ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [77ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [80ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1718304909(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d248163 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1686439836 after retry RPC@1837292714) -> (continuation of Deferred@122596301 after retry RPC@2094157826), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@481a1ff -> wakeup thread Time-limited test -> (continuation of Deferred@1686439836 after retry RPC@1837292714) -> (continuation of Deferred@122596301 after retry RPC@2094157826))) after lookup
08:03:40.493 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:40.494 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.494 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.494 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.571 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:40.572 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:40.572 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=161), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [2ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [20ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [22ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [22ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [40ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [42ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [42ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [77ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [80ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [81ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [83ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [84ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [158ms] refreshing cache from master, [158ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [160ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1718304909(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d248163 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1686439836 after retry RPC@1837292714) -> (continuation of Deferred@122596301 after retry RPC@2094157826) -> (continuation of Deferred@1948936354 after retry RPC@446067813), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@481a1ff -> wakeup thread Time-limited test -> (continuation of Deferred@1686439836 after retry RPC@1837292714) -> (continuation of Deferred@122596301 after retry RPC@2094157826) -> (continuation of Deferred@1948936354 after retry RPC@446067813))) after lookup
08:03:40.573 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:40.574 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.574 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.574 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.870 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:40.871 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:40.871 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=460), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [2ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [20ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [22ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [22ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [40ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [42ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [42ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [77ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [80ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [81ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [83ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [84ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [158ms] refreshing cache from master, [158ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [160ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [161ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [163ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [163ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [457ms] refreshing cache from master, [457ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [459ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1718304909(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d248163 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1686439836 after retry RPC@1837292714) -> (continuation of Deferred@122596301 after retry RPC@2094157826) -> (continuation of Deferred@1948936354 after retry RPC@446067813) -> (continuation of Deferred@1611790817 after retry RPC@1225624794), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@481a1ff -> wakeup thread Time-limited test -> (continuation of Deferred@1686439836 after retry RPC@1837292714) -> (continuation of Deferred@122596301 after retry RPC@2094157826) -> (continuation of Deferred@1948936354 after retry RPC@446067813) -> (continuation of Deferred@1611790817 after retry RPC@1225624794))) after lookup
08:03:40.872 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:40.873 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.873 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:40.873 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:41.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.709318 18763 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 010d2db890fc482bb61ce7809297487b)
08:03:41.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.709476 18763 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:41.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.710438 18763 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:41.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:41.711086 18466 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:41.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:41.711869 18466 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.715215 18704 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" is_pre_election: true
08:03:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.715451 18704 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 13ed5f47b5c14458a6cc3efb8918cd2c in term 6.
08:03:41.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.716102 18465 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 13ed5f47b5c14458a6cc3efb8918cd2c, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 010d2db890fc482bb61ce7809297487b
08:03:41.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.716331 18763 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 7 FOLLOWER]: Leader pre-election won for term 8
08:03:41.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.716385 18763 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 7 FOLLOWER]: Starting leader election (detected failure of leader 010d2db890fc482bb61ce7809297487b)
08:03:41.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.716413 18763 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 7 FOLLOWER]: Advancing to term 8
08:03:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.717149 18763 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.717432 18763 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 8 election: Requested vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.717789 18704 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.717917 18704 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 6 FOLLOWER]: Advancing to term 8
08:03:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:41.718359 18466 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:41.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.719566 18704 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 13ed5f47b5c14458a6cc3efb8918cd2c in term 8.
08:03:41.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.719842 18465 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 13ed5f47b5c14458a6cc3efb8918cd2c, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 010d2db890fc482bb61ce7809297487b
08:03:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.719988 18763 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 8 FOLLOWER]: Leader election won for term 8
08:03:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.720153 18763 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 8 LEADER]: Becoming Leader. State: Replica: 13ed5f47b5c14458a6cc3efb8918cd2c, State: Running, Role: LEADER
08:03:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.720429 18763 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:41.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:41.722656 17170 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c reported cstate change: term changed from 7 to 8, leader changed from 010d2db890fc482bb61ce7809297487b (127.16.186.67) to 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65). New cstate: current_term: 8 leader_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: HEALTHY } } }
08:03:42.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.179159 18466 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c -> Peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Couldn't send request to peer 010d2db890fc482bb61ce7809297487b. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.272118 18704 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 8 FOLLOWER]: Refusing update from remote peer 13ed5f47b5c14458a6cc3efb8918cd2c: Log matching property violated. Preceding OpId in replica: term: 6 index: 20. Preceding OpId from leader: term: 8 index: 25. (index mismatch)
08:03:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.272801 18763 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
08:03:42.611 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:42.611 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:42.612 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2200), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [2ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [20ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [22ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [22ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [40ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [42ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [42ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [77ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [80ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [81ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [83ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [84ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [158ms] refreshing cache from master, [158ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [160ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [161ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [163ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [163ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [457ms] refreshing cache from master, [457ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [459ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [461ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [462ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [462ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [2197ms] refreshing cache from master, [2197ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [2200ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1718304909(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@d248163 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1686439836 after retry RPC@1837292714) -> (continuation of Deferred@122596301 after retry RPC@2094157826) -> (continuation of Deferred@1948936354 after retry RPC@446067813) -> (continuation of Deferred@1611790817 after retry RPC@1225624794) -> (continuation of Deferred@1630290984 after retry RPC@1155690379), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@481a1ff -> wakeup thread Time-limited test -> (continuation of Deferred@1686439836 after retry RPC@1837292714) -> (continuation of Deferred@122596301 after retry RPC@2094157826) -> (continuation of Deferred@1948936354 after retry RPC@446067813) -> (continuation of Deferred@1611790817 after retry RPC@1225624794) -> (continuation of Deferred@1630290984 after retry RPC@1155690379))) after lookup
08:03:42.612 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:42.613 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] Successfully connected to peer
08:03:42.628 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x822e9ff2, L:/127.0.0.1:51714 - R:/127.16.186.65:40953] using TOKEN/null
08:03:42.644 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.186.67:41083
08:03:42.644 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.645108 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/logs
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:42.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.67:41083
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.67
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39371
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.67
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:42.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.746860 18791 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:42.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.747052 18791 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:42.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.747097 18791 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.750401 18791 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.750504 18791 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.67
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.754245 18791 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.67:41083
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.67
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39371
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18791
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.67
08:03:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/logs
08:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.755183 18791 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:42.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.756214 18791 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.762069 18801 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.762149 18798 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.762248 18791 server_base.cc:1061] running on GCE node
08:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:42.762112 18799 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.762739 18791 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.763270 18791 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.764429 18791 hybrid_clock.cc:648] HybridClock initialized: now 1779091422764418 us; error 23 us; skew 500 ppm
08:03:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.766096 18791 webserver.cc:533] Webserver started at http://127.16.186.67:39371/ using document root <none> and password file <none>
08:03:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.766647 18791 fs_manager.cc:362] Metadata directory not provided
08:03:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.766753 18791 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:42.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.769865 18791 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.000s	sys 0.003s
08:03:42.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.771561 18807 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.772401 18791 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.772505 18791 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010d2db890fc482bb61ce7809297487b"
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:22 on dist-test-slave-lw51"
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.772899 18791 fs_report.cc:389] FS layout report
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/data
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:42.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.785331 18791 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.785934 18791 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.786096 18791 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.786674 18791 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.787827 18814 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
08:03:42.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.790858 18791 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:42.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.790915 18791 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
08:03:42.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.790971 18791 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:42.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.792346 18791 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:42.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.792387 18791 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
08:03:42.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.792652 18814 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Bootstrap starting.
08:03:42.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.802395 18791 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.67:41083
08:03:42.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.803272 18921 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.67:41083 every 8 connection(s)
08:03:42.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.803575 18791 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.808918 18814 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Log is configured to *not* fsync() on all Append() calls
08:03:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.810527 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 18791
08:03:42.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.814220 18924 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.814440 18924 heartbeater.cc:461] Registering TS with master...
08:03:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.815006 18924 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.818493 18923 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.818576 18923 heartbeater.cc:461] Registering TS with master...
08:03:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.818773 18923 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.819610 17242 ts_manager.cc:194] Re-registered known tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.819617 17170 ts_manager.cc:194] Re-registered known tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:42.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.820482 17170 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.67:59615
08:03:42.820 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:42.821 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.186.65:40953
08:03:42.821 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:03:42.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.822227 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 18448
08:03:42.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.827836 18814 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:42.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.828203 18814 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Bootstrap complete.
08:03:42.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.828863 18922 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:42.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.828971 18922 heartbeater.cc:461] Registering TS with master...
08:03:42.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.829156 18922 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:42.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.829299 18814 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Time spent bootstrapping tablet: real 0.037s	user 0.023s	sys 0.009s
08:03:42.829 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] handling channelInactive
08:03:42.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state READY due to: connection closed
08:03:42.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.830128 17313 ts_manager.cc:194] Re-registered known tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:42.830 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:42.831 [DEBUG - Time-limited test] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:42.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.831946 18814 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:42.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.832266 18814 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 010d2db890fc482bb61ce7809297487b, State: Initialized, Role: FOLLOWER
08:03:42.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.832 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.832686 18814 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:42.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.833256 18923 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:42.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:42.833503 18814 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Time spent starting tablet: real 0.004s	user 0.008s	sys 0.000s
08:03:42.851 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:42.851 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:42.851 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [1ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@979345722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@698184f8 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20327b24 -> wakeup thread Time-limited test)) after lookup
08:03:42.852 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:42.853 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.853 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.853 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.871 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:42.872 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:42.872 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [1ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [20ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [40ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@979345722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@698184f8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@107852373 after retry RPC@1291735791), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20327b24 -> wakeup thread Time-limited test -> (continuation of Deferred@107852373 after retry RPC@1291735791))) after lookup
08:03:42.873 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:42.874 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.874 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.874 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.890 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:42.891 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:42.891 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=60), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [1ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [20ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [40ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [42ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [43ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [43ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [59ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@979345722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@698184f8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@107852373 after retry RPC@1291735791) -> (continuation of Deferred@1366741690 after retry RPC@1864469329), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20327b24 -> wakeup thread Time-limited test -> (continuation of Deferred@107852373 after retry RPC@1291735791) -> (continuation of Deferred@1366741690 after retry RPC@1864469329))) after lookup
08:03:42.892 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:42.892 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.893 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.893 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.950 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:42.951 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:42.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=120), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [1ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [20ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [40ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [42ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [43ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [43ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [59ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [60ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [62ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [62ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [119ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@979345722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@698184f8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@107852373 after retry RPC@1291735791) -> (continuation of Deferred@1366741690 after retry RPC@1864469329) -> (continuation of Deferred@1981827261 after retry RPC@364396075), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20327b24 -> wakeup thread Time-limited test -> (continuation of Deferred@107852373 after retry RPC@1291735791) -> (continuation of Deferred@1366741690 after retry RPC@1864469329) -> (continuation of Deferred@1981827261 after retry RPC@364396075))) after lookup
08:03:42.952 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:42.953 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.953 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:42.953 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:43.391 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:43.391 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:43.391 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=560), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [1ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [20ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [40ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [42ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [43ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [43ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [59ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [60ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [62ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [62ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [119ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [120ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [122ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [122ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [557ms] refreshing cache from master, [557ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [559ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@979345722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@698184f8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@107852373 after retry RPC@1291735791) -> (continuation of Deferred@1366741690 after retry RPC@1864469329) -> (continuation of Deferred@1981827261 after retry RPC@364396075) -> (continuation of Deferred@720280428 after retry RPC@2129895804), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20327b24 -> wakeup thread Time-limited test -> (continuation of Deferred@107852373 after retry RPC@1291735791) -> (continuation of Deferred@1366741690 after retry RPC@1864469329) -> (continuation of Deferred@1981827261 after retry RPC@364396075) -> (continuation of Deferred@720280428 after retry RPC@2129895804))) after lookup
08:03:43.392 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:43.394 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:43.394 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:43.394 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:44.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.144888 18936 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 13ed5f47b5c14458a6cc3efb8918cd2c)
08:03:44.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.145023 18936 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:44.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.146004 18936 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:44.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.146689 18639 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:44.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.147491 18639 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:44.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.150770 18876 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b" is_pre_election: true
08:03:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.151021 18876 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3782bdbeb06c497ca787a0b7b2e6c918 in term 7.
08:03:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.151638 18638 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:44.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.151870 18936 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 8 FOLLOWER]: Leader pre-election won for term 9
08:03:44.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.151939 18936 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 13ed5f47b5c14458a6cc3efb8918cd2c)
08:03:44.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.151980 18936 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 8 FOLLOWER]: Advancing to term 9
08:03:44.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.152772 18936 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:44.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.153121 18936 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 9 election: Requested vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:44.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.153429 18876 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b"
08:03:44.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.153571 18876 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 7 FOLLOWER]: Advancing to term 9
08:03:44.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.154047 18639 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.155228 18876 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3782bdbeb06c497ca787a0b7b2e6c918 in term 9.
08:03:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.155521 18638 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.155680 18936 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 9 FOLLOWER]: Leader election won for term 9
08:03:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.155889 18936 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 9 LEADER]: Becoming Leader. State: Replica: 3782bdbeb06c497ca787a0b7b2e6c918, State: Running, Role: LEADER
08:03:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.156168 18936 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.158428 17170 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 reported cstate change: term changed from 8 to 9, leader changed from 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65) to 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66). New cstate: current_term: 9 leader_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: UNKNOWN } } }
08:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.581243 18639 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 -> Peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Couldn't send request to peer 13ed5f47b5c14458a6cc3efb8918cd2c. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.651881 18876 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 9 FOLLOWER]: Refusing update from remote peer 3782bdbeb06c497ca787a0b7b2e6c918: Log matching property violated. Preceding OpId in replica: term: 7 index: 24. Preceding OpId from leader: term: 9 index: 29. (index mismatch)
08:03:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.652539 18936 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [LEADER]: Connected to new peer: Peer: permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
08:03:44.831 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:44.831 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:44.831 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2000), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [1ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [20ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [22ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [22ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [40ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [42ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [43ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [43ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [59ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [60ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [62ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [62ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [119ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [120ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [122ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [122ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [557ms] refreshing cache from master, [557ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [559ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [561ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [563ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [563ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1997ms] refreshing cache from master, [1997ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [1999ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@979345722(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@698184f8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@107852373 after retry RPC@1291735791) -> (continuation of Deferred@1366741690 after retry RPC@1864469329) -> (continuation of Deferred@1981827261 after retry RPC@364396075) -> (continuation of Deferred@720280428 after retry RPC@2129895804) -> (continuation of Deferred@1521599644 after retry RPC@959084279), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20327b24 -> wakeup thread Time-limited test -> (continuation of Deferred@107852373 after retry RPC@1291735791) -> (continuation of Deferred@1366741690 after retry RPC@1864469329) -> (continuation of Deferred@1981827261 after retry RPC@364396075) -> (continuation of Deferred@720280428 after retry RPC@2129895804) -> (continuation of Deferred@1521599644 after retry RPC@959084279))) after lookup
08:03:44.832 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:44.833 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] Successfully connected to peer
08:03:44.854 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x4ff435af, L:/127.0.0.1:37716 - R:/127.16.186.66:46109] using TOKEN/null
08:03:44.869 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.186.65:40953
08:03:44.870 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:03:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.870597 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data
08:03:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/logs
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.65:40953
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.65
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39271
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.65
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:44.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.966542 18965 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:44.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.966725 18965 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:44.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.966765 18965 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:44.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.969910 18965 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:44.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.970001 18965 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.65
08:03:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.973582 18965 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data
08:03:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.65:40953
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.65
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39271
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18965
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.65
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/logs
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.974478 18965 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:44.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.975432 18965 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:44.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.981168 18971 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:44.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.981173 18973 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:44.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.981499 18965 server_base.cc:1061] running on GCE node
08:03:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:44.981959 18970 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.982295 18965 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.982786 18965 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.983937 18965 hybrid_clock.cc:648] HybridClock initialized: now 1779091424983925 us; error 23 us; skew 500 ppm
08:03:44.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.985548 18965 webserver.cc:533] Webserver started at http://127.16.186.65:39271/ using document root <none> and password file <none>
08:03:44.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.986073 18965 fs_manager.cc:362] Metadata directory not provided
08:03:44.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.986127 18965 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:44.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.989104 18965 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.000s	sys 0.003s
08:03:44.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.990803 18979 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:44.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.991686 18965 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.991771 18965 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13ed5f47b5c14458a6cc3efb8918cd2c"
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:44.992136 18965 fs_report.cc:389] FS layout report
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/data
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:44.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:44.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.010453 18965 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:45.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.011024 18965 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:45.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.011169 18965 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:45.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.011621 18965 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:45.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.012732 18986 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
08:03:45.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.015914 18965 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:45.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.015969 18965 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
08:03:45.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.016021 18965 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.017717 18965 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.017766 18965 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.002s	sys 0.000s
08:03:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.018113 18986 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap starting.
08:03:45.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.027696 18965 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.65:40953
08:03:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.028234 19093 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.65:40953 every 8 connection(s)
08:03:45.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.029095 18965 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.035418 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 18965
08:03:45.035 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.035890 18986 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Log is configured to *not* fsync() on all Append() calls
08:03:45.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.043329 19094 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:45.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.043563 19094 heartbeater.cc:461] Registering TS with master...
08:03:45.043 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:45.044 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.186.66:46109
08:03:45.044 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:03:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.044132 19094 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.044714 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 18620
08:03:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.045542 19096 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.045615 19096 heartbeater.cc:461] Registering TS with master...
08:03:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.045804 19096 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.046165 19095 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.046233 19095 heartbeater.cc:461] Registering TS with master...
08:03:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.046386 19095 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.047107 17242 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:45.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.047703 17171 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:45.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.047757 17313 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:45.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.048612 17171 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.65:38821
08:03:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:45.051432 18981 connection.cc:582] server connection from 127.16.186.66:53737 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:45.052 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:45.052 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] handling channelInactive
08:03:45.052 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state READY due to: connection closed
08:03:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.061193 18986 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.061618 18986 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap complete.
08:03:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.062463 18986 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Time spent bootstrapping tablet: real 0.045s	user 0.025s	sys 0.010s
08:03:45.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.064438 18986 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:45.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.064738 18986 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 13ed5f47b5c14458a6cc3efb8918cd2c, State: Initialized, Role: FOLLOWER
08:03:45.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.065174 18986 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:45.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.065804 19095 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:45.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:45.066007 18986 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Time spent starting tablet: real 0.003s	user 0.004s	sys 0.000s
08:03:45.068 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:45.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.070 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.090 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:45.091 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:45.091 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=3, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [18ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [18ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@342211268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@7b07e677 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44a5b3c7 -> wakeup thread Time-limited test)) after lookup
08:03:45.091 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:45.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.092 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.111 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:45.111 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:45.111 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=5, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [18ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [18ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [39ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [40ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [40ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [56ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@342211268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@7b07e677 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1519591965 after retry RPC@868321539), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44a5b3c7 -> wakeup thread Time-limited test -> (continuation of Deferred@1519591965 after retry RPC@868321539))) after lookup
08:03:45.112 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:45.113 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.113 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.113 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.191 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:45.191 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:45.191 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=7, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=139), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [18ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [18ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [39ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [40ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [40ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [56ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [60ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [61ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [61ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [136ms] refreshing cache from master, [136ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [139ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@342211268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@7b07e677 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1519591965 after retry RPC@868321539) -> (continuation of Deferred@411997357 after retry RPC@666406702), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44a5b3c7 -> wakeup thread Time-limited test -> (continuation of Deferred@1519591965 after retry RPC@868321539) -> (continuation of Deferred@411997357 after retry RPC@666406702))) after lookup
08:03:45.192 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:45.193 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.193 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.193 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.371 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:45.371 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:45.372 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=9, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=319), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [18ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [18ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [39ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [40ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [40ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [56ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [60ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [61ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [61ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [136ms] refreshing cache from master, [136ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [139ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [140ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [141ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [141ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [316ms] refreshing cache from master, [316ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [319ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@342211268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@7b07e677 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1519591965 after retry RPC@868321539) -> (continuation of Deferred@411997357 after retry RPC@666406702) -> (continuation of Deferred@249709931 after retry RPC@1832938086), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44a5b3c7 -> wakeup thread Time-limited test -> (continuation of Deferred@1519591965 after retry RPC@868321539) -> (continuation of Deferred@411997357 after retry RPC@666406702) -> (continuation of Deferred@249709931 after retry RPC@1832938086))) after lookup
08:03:45.373 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:45.374 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.374 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.374 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.671 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:45.671 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:45.671 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=11, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=619), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [18ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [18ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [39ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [40ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [40ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [56ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [60ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [61ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [61ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [136ms] refreshing cache from master, [136ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [139ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [140ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [141ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [141ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [316ms] refreshing cache from master, [316ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [319ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [320ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [322ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [322ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [616ms] refreshing cache from master, [616ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [618ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@342211268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@7b07e677 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1519591965 after retry RPC@868321539) -> (continuation of Deferred@411997357 after retry RPC@666406702) -> (continuation of Deferred@249709931 after retry RPC@1832938086) -> (continuation of Deferred@1094391289 after retry RPC@801522914), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44a5b3c7 -> wakeup thread Time-limited test -> (continuation of Deferred@1519591965 after retry RPC@868321539) -> (continuation of Deferred@411997357 after retry RPC@666406702) -> (continuation of Deferred@249709931 after retry RPC@1832938086) -> (continuation of Deferred@1094391289 after retry RPC@801522914))) after lookup
08:03:45.672 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:45.673 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.673 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state CONNECTING due to: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:45.673 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109
08:03:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.349557 19108 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 8 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.349713 19108 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.350673 19108 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:46.351372 18981 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111)
08:03:46.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:46.352218 18981 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109): Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111)
08:03:46.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.354622 18876 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b" is_pre_election: true
08:03:46.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.355122 18982 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 13ed5f47b5c14458a6cc3efb8918cd2c; no voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918
08:03:46.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.355348 19108 raft_consensus.cc:2749] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 8 FOLLOWER]: Leader pre-election lost for term 9. Reason: could not achieve majority
08:03:46.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.369145 19113 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 3782bdbeb06c497ca787a0b7b2e6c918)
08:03:46.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.369259 19113 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:46.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.370236 19113 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:46.370913 18809 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111)
08:03:46.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:46.372022 18809 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109): Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111)
08:03:46.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.374203 19048 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "010d2db890fc482bb61ce7809297487b" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" is_pre_election: true
08:03:46.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.374392 19048 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 010d2db890fc482bb61ce7809297487b in term 8.
08:03:46.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.374720 18811 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c; no voters: 3782bdbeb06c497ca787a0b7b2e6c918
08:03:46.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.374907 19113 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 9 FOLLOWER]: Leader pre-election won for term 10
08:03:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.374963 19113 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 9 FOLLOWER]: Starting leader election (detected failure of leader 3782bdbeb06c497ca787a0b7b2e6c918)
08:03:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.375000 19113 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 9 FOLLOWER]: Advancing to term 10
08:03:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.375715 19113 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.376008 19113 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 10 election: Requested vote from peers 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.376474 19048 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "010d2db890fc482bb61ce7809297487b" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c"
08:03:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.376607 19048 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 8 FOLLOWER]: Advancing to term 10
08:03:46.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:46.376916 18809 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109): Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111)
08:03:46.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.378449 19048 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 010d2db890fc482bb61ce7809297487b in term 10.
08:03:46.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.378712 18811 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c; no voters: 3782bdbeb06c497ca787a0b7b2e6c918
08:03:46.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.378930 19113 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 10 FOLLOWER]: Leader election won for term 10
08:03:46.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.379135 19113 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 10 LEADER]: Becoming Leader. State: Replica: 010d2db890fc482bb61ce7809297487b, State: Running, Role: LEADER
08:03:46.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.379428 19113 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:46.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.381815 17171 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b reported cstate change: term changed from 9 to 10, leader changed from 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66) to 010d2db890fc482bb61ce7809297487b (127.16.186.67). New cstate: current_term: 10 leader_uuid: "010d2db890fc482bb61ce7809297487b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: UNKNOWN } } }
08:03:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:46.870350 18809 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b -> Peer 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109): Couldn't send request to peer 3782bdbeb06c497ca787a0b7b2e6c918. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.66:46109: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.871155 19048 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 10 FOLLOWER]: Refusing update from remote peer 010d2db890fc482bb61ce7809297487b: Log matching property violated. Preceding OpId in replica: term: 8 index: 28. Preceding OpId from leader: term: 10 index: 33. (index mismatch)
08:03:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:46.871713 19113 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [LEADER]: Connected to new peer: Peer: permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
08:03:49.052 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:49.052 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:49.052 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=13, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=4000), Traces: [0ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [18ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [18ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [38ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [39ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [40ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [40ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [56ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [60ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [61ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [61ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [136ms] refreshing cache from master, [136ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [139ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [140ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [141ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [141ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [316ms] refreshing cache from master, [316ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [319ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [320ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [322ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [322ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [616ms] refreshing cache from master, [616ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [618ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [620ms] sending RPC to server 3782bdbeb06c497ca787a0b7b2e6c918, [621ms] received response from server 3782bdbeb06c497ca787a0b7b2e6c918: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [621ms] delaying RPC due to: Network error: Failed to connect to peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109): Connection refused: /127.16.186.66:46109, [3997ms] refreshing cache from master, [3997ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [3999ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@342211268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@7b07e677 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1519591965 after retry RPC@868321539) -> (continuation of Deferred@411997357 after retry RPC@666406702) -> (continuation of Deferred@249709931 after retry RPC@1832938086) -> (continuation of Deferred@1094391289 after retry RPC@801522914) -> (continuation of Deferred@965412968 after retry RPC@611826415), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44a5b3c7 -> wakeup thread Time-limited test -> (continuation of Deferred@1519591965 after retry RPC@868321539) -> (continuation of Deferred@411997357 after retry RPC@666406702) -> (continuation of Deferred@249709931 after retry RPC@1832938086) -> (continuation of Deferred@1094391289 after retry RPC@801522914) -> (continuation of Deferred@965412968 after retry RPC@611826415))) after lookup
08:03:49.054 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:49.055 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] Successfully connected to peer
08:03:49.076 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x43047c4a, L:/127.0.0.1:58410 - R:/127.16.186.67:41083] using TOKEN/null
08:03:49.092 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.186.66:46109
08:03:49.093 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.093844 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/logs
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.66:46109
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.66
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37919
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.66
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.194993 19148 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.195204 19148 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.195263 19148 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.198645 19148 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.198767 19148 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.66
08:03:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.202556 19148 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.66:46109
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.66
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37919
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.19148
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.66
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/logs
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.203929 19148 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.205050 19148 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.211181 19156 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.211144 19153 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:49.211165 19154 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.211226 19148 server_base.cc:1061] running on GCE node
08:03:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.211823 19148 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.212327 19148 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.213492 19148 hybrid_clock.cc:648] HybridClock initialized: now 1779091429213472 us; error 34 us; skew 500 ppm
08:03:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.215098 19148 webserver.cc:533] Webserver started at http://127.16.186.66:37919/ using document root <none> and password file <none>
08:03:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.215615 19148 fs_manager.cc:362] Metadata directory not provided
08:03:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.215672 19148 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.218878 19148 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
08:03:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.220544 19162 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.221390 19148 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.221508 19148 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.221876 19148 fs_report.cc:389] FS layout report
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/wal
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/data
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.237635 19148 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:49.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.238304 19148 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:49.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.238473 19148 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:49.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.239029 19148 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:49.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.240216 19169 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
08:03:49.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.243306 19148 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:49.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.243359 19148 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
08:03:49.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.243415 19148 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:49.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.244822 19148 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:49.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.244867 19148 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
08:03:49.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.245163 19169 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Bootstrap starting.
08:03:49.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.254509 19148 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.66:46109
08:03:49.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.254549 19276 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.66:46109 every 8 connection(s)
08:03:49.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.255405 19148 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-1/data/info.pb
08:03:49.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.259459 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 19148
08:03:49.259 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.264492 19169 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Log is configured to *not* fsync() on all Append() calls
08:03:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.264753 19279 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.264971 19279 heartbeater.cc:461] Registering TS with master...
08:03:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.265571 19279 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:49.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.266045 19277 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:49.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.266129 19277 heartbeater.cc:461] Registering TS with master...
08:03:49.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.266317 19277 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:49.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.267315 17313 ts_manager.cc:194] Re-registered known tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:49.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.267316 17242 ts_manager.cc:194] Re-registered known tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:49.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.268642 19278 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.268713 19278 heartbeater.cc:461] Registering TS with master...
08:03:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.268883 19278 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:49.270 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.269624 17170 ts_manager.cc:194] Re-registered known tserver with Master: 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.270499 17170 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.66:45195
08:03:49.272 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.186.67:41083
08:03:49.272 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:03:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.272647 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 18791
08:03:49.278 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:49.279 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] handling channelInactive
08:03:49.279 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state READY due to: connection closed
08:03:49.279 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: connection closed
08:03:49.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.285676 19169 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:49.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.286007 19169 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Bootstrap complete.
08:03:49.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.286782 19169 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Time spent bootstrapping tablet: real 0.042s	user 0.024s	sys 0.011s
08:03:49.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.288693 19169 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:49.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.289036 19169 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3782bdbeb06c497ca787a0b7b2e6c918, State: Initialized, Role: FOLLOWER
08:03:49.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.289528 19169 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:49.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.290282 19278 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:49.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:49.290496 19169 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
08:03:49.291 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:49.292 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:49.292 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [0ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [12ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1231173800(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@505b43a9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4724a287 -> wakeup thread Time-limited test)) after lookup
08:03:49.292 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:49.293 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.294 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.294 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.311 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:49.311 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:49.312 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [0ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [12ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [13ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [15ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [15ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1231173800(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@505b43a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4724a287 -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431))) after lookup
08:03:49.312 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:49.313 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.314 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.314 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.350 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:49.350 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:49.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=71), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [0ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [12ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [13ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [15ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [15ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [33ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [35ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [35ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [71ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1231173800(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@505b43a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431) -> (continuation of Deferred@2007669994 after retry RPC@1131906588), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4724a287 -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431) -> (continuation of Deferred@2007669994 after retry RPC@1131906588))) after lookup
08:03:49.351 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:49.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.352 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.411 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:49.411 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:49.411 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=132), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [0ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [12ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [13ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [15ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [15ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [33ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [35ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [35ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [71ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [72ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [73ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [73ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [129ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [132ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1231173800(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@505b43a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431) -> (continuation of Deferred@2007669994 after retry RPC@1131906588) -> (continuation of Deferred@1484164475 after retry RPC@667854607), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4724a287 -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431) -> (continuation of Deferred@2007669994 after retry RPC@1131906588) -> (continuation of Deferred@1484164475 after retry RPC@667854607))) after lookup
08:03:49.412 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:49.413 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.413 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.414 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.671 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:49.671 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:49.672 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=392), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [0ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [12ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [13ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [15ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [15ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [33ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [35ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [35ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [71ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [72ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [73ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [73ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [129ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [132ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [133ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [135ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [135ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [389ms] refreshing cache from master, [390ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [392ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1231173800(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@505b43a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431) -> (continuation of Deferred@2007669994 after retry RPC@1131906588) -> (continuation of Deferred@1484164475 after retry RPC@667854607) -> (continuation of Deferred@319234788 after retry RPC@1975847183), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4724a287 -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431) -> (continuation of Deferred@2007669994 after retry RPC@1131906588) -> (continuation of Deferred@1484164475 after retry RPC@667854607) -> (continuation of Deferred@319234788 after retry RPC@1975847183))) after lookup
08:03:49.673 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:49.674 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.674 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:49.674 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:50.091 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:50.091 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)[L],13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:50.092 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=812), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [0ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [12ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [13ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [15ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [15ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [33ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [35ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [35ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [71ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [72ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [73ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [73ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [129ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [132ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [133ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [135ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [135ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [389ms] refreshing cache from master, [390ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [392ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [393ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [395ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [395ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [809ms] refreshing cache from master, [810ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [812ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1231173800(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@505b43a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431) -> (continuation of Deferred@2007669994 after retry RPC@1131906588) -> (continuation of Deferred@1484164475 after retry RPC@667854607) -> (continuation of Deferred@319234788 after retry RPC@1975847183) -> (continuation of Deferred@749282818 after retry RPC@1903812585), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4724a287 -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431) -> (continuation of Deferred@2007669994 after retry RPC@1131906588) -> (continuation of Deferred@1484164475 after retry RPC@667854607) -> (continuation of Deferred@319234788 after retry RPC@1975847183) -> (continuation of Deferred@749282818 after retry RPC@1903812585))) after lookup
08:03:50.092 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] connecting to peer
08:03:50.093 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:50.093 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:50.093 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083
08:03:50.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.461033 19290 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:50.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.461221 19290 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:50.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.462240 19290 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:50.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:50.462927 19165 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:50.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:50.463701 19165 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.465987 19048 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" is_pre_election: true
08:03:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.466452 19166 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 010d2db890fc482bb61ce7809297487b, 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.466660 19290 raft_consensus.cc:2749] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
08:03:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.592813 19295 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 010d2db890fc482bb61ce7809297487b)
08:03:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.592947 19295 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.593436 19295 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:50.594616 18982 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:50.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.597805 19231 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" is_pre_election: true
08:03:50.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.598022 19231 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 13ed5f47b5c14458a6cc3efb8918cd2c in term 9.
08:03:50.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.598625 18981 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 13ed5f47b5c14458a6cc3efb8918cd2c, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 010d2db890fc482bb61ce7809297487b
08:03:50.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.598794 19295 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 10 FOLLOWER]: Leader pre-election won for term 11
08:03:50.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.598850 19295 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 10 FOLLOWER]: Starting leader election (detected failure of leader 010d2db890fc482bb61ce7809297487b)
08:03:50.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.598879 19295 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 10 FOLLOWER]: Advancing to term 11
08:03:50.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.599596 19295 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:50.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.599885 19295 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 11 election: Requested vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66:46109)
08:03:50.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.600272 19231 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "3782bdbeb06c497ca787a0b7b2e6c918"
08:03:50.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.600394 19231 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 9 FOLLOWER]: Advancing to term 11
08:03:50.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:50.600729 18982 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111)
08:03:50.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.602057 19231 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 13ed5f47b5c14458a6cc3efb8918cd2c in term 11.
08:03:50.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.602322 18981 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 13ed5f47b5c14458a6cc3efb8918cd2c, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 010d2db890fc482bb61ce7809297487b
08:03:50.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.602463 19295 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 11 FOLLOWER]: Leader election won for term 11
08:03:50.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.602629 19295 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 11 LEADER]: Becoming Leader. State: Replica: 13ed5f47b5c14458a6cc3efb8918cd2c, State: Running, Role: LEADER
08:03:50.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.603058 19295 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:50.605631 17170 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c reported cstate change: term changed from 10 to 11, leader changed from 010d2db890fc482bb61ce7809297487b (127.16.186.67) to 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65). New cstate: current_term: 11 leader_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: HEALTHY } } }
08:03:51.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.122231 18982 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c -> Peer 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083): Couldn't send request to peer 010d2db890fc482bb61ce7809297487b. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.67:41083: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:51.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.191252 19231 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 11 FOLLOWER]: Refusing update from remote peer 13ed5f47b5c14458a6cc3efb8918cd2c: Log matching property violated. Preceding OpId in replica: term: 9 index: 32. Preceding OpId from leader: term: 11 index: 37. (index mismatch)
08:03:51.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.191829 19313 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
08:03:51.270 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:51.271 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:51.271 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1992), Traces: [0ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [0ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: connection closed, [0ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [12ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [13ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [15ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [15ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [33ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [35ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [35ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [71ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [72ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [73ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [73ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [129ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [132ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [133ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [135ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [135ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [389ms] refreshing cache from master, [390ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [392ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [393ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [395ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [395ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [809ms] refreshing cache from master, [810ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [812ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [813ms] sending RPC to server 010d2db890fc482bb61ce7809297487b, [814ms] received response from server 010d2db890fc482bb61ce7809297487b: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [814ms] delaying RPC due to: Network error: Failed to connect to peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083): Connection refused: /127.16.186.67:41083, [1989ms] refreshing cache from master, [1989ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [1991ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1231173800(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@505b43a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431) -> (continuation of Deferred@2007669994 after retry RPC@1131906588) -> (continuation of Deferred@1484164475 after retry RPC@667854607) -> (continuation of Deferred@319234788 after retry RPC@1975847183) -> (continuation of Deferred@749282818 after retry RPC@1903812585) -> (continuation of Deferred@1828049035 after retry RPC@1405930603), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4724a287 -> wakeup thread Time-limited test -> (continuation of Deferred@1623564714 after retry RPC@43456431) -> (continuation of Deferred@2007669994 after retry RPC@1131906588) -> (continuation of Deferred@1484164475 after retry RPC@667854607) -> (continuation of Deferred@319234788 after retry RPC@1975847183) -> (continuation of Deferred@749282818 after retry RPC@1903812585) -> (continuation of Deferred@1828049035 after retry RPC@1405930603))) after lookup
08:03:51.272 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:51.273 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] Successfully connected to peer
08:03:51.293 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0xe7351275, L:/127.0.0.1:58660 - R:/127.16.186.65:40953] using TOKEN/null
08:03:51.307 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.186.67:41083
08:03:51.307 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.308230 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/logs
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.67:41083
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.67
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39371
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.67
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:51.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:51.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.406397 19327 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:51.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.406601 19327 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:51.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.406646 19327 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:51.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.409901 19327 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:51.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.410000 19327 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.67
08:03:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.413790 19327 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.67:41083
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.67
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39371
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.19327
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.67
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/logs
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.414795 19327 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:51.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.415835 19327 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.421746 19332 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.421773 19333 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:51.421759 19335 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.421962 19327 server_base.cc:1061] running on GCE node
08:03:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.422550 19327 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:51.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.423060 19327 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:51.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.424227 19327 hybrid_clock.cc:648] HybridClock initialized: now 1779091431424210 us; error 24 us; skew 500 ppm
08:03:51.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.425933 19327 webserver.cc:533] Webserver started at http://127.16.186.67:39371/ using document root <none> and password file <none>
08:03:51.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.426491 19327 fs_manager.cc:362] Metadata directory not provided
08:03:51.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.426550 19327 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:51.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.429668 19327 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
08:03:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.431413 19341 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:51.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.432371 19327 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
08:03:51.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.432471 19327 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:51.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010d2db890fc482bb61ce7809297487b"
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:22 on dist-test-slave-lw51"
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.432853 19327 fs_report.cc:389] FS layout report
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/wal
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/data
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.444885 19327 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.445401 19327 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.445570 19327 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:51.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.446014 19327 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:51.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.447086 19348 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
08:03:51.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.450117 19327 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:51.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.450177 19327 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
08:03:51.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.450230 19327 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:51.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.451534 19327 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:51.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.451573 19327 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
08:03:51.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.451856 19348 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Bootstrap starting.
08:03:51.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.461297 19327 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.67:41083
08:03:51.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.461333 19455 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.67:41083 every 8 connection(s)
08:03:51.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.462194 19327 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-2/data/info.pb
08:03:51.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.463083 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 19327
08:03:51.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.468896 19348 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Log is configured to *not* fsync() on all Append() calls
08:03:51.472 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:51.473 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.186.65:40953
08:03:51.473 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:03:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.473637 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 18965
08:03:51.479 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] handling channelInactive
08:03:51.479 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:51.480 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state READY due to: connection closed
08:03:51.480 [DEBUG - Time-limited test] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:51.481 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.481 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.482 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.491 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:51.491 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:51.492 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [2ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [2ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1728297252(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@52198e3b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c537b23 -> wakeup thread Time-limited test)) after lookup
08:03:51.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.492612 19348 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:51.492 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:51.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.493094 19348 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Bootstrap complete.
08:03:51.493 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.494 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.494 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.494092 19348 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Time spent bootstrapping tablet: real 0.042s	user 0.023s	sys 0.015s
08:03:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.496407 19348 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.496838 19348 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 010d2db890fc482bb61ce7809297487b, State: Initialized, Role: FOLLOWER
08:03:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.497319 19348 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.497802 19456 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.498045 19456 heartbeater.cc:461] Registering TS with master...
08:03:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.498251 19348 ts_tablet_manager.cc:1434] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
08:03:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.498556 19458 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.498636 19458 heartbeater.cc:461] Registering TS with master...
08:03:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.498663 19456 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.498818 19458 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.499729 19457 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.499799 19457 heartbeater.cc:461] Registering TS with master...
08:03:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.499979 19457 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.500449 17242 ts_manager.cc:194] Re-registered known tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.500449 17313 ts_manager.cc:194] Re-registered known tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:51.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.501224 17171 ts_manager.cc:194] Re-registered known tserver with Master: 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083)
08:03:51.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.502065 17171 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.67:51699
08:03:51.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:51.503633 19457 heartbeater.cc:499] Master 127.16.186.126:40625 was elected leader, sending a full tablet report...
08:03:51.510 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:51.511 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:51.511 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [2ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [2ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [12ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [14ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [14ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [30ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1728297252(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@52198e3b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c537b23 -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046))) after lookup
08:03:51.511 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:51.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.512 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.531 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:51.531 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:51.532 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=51), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [2ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [2ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [12ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [14ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [14ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [30ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [31ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [32ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [32ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [48ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1728297252(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@52198e3b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046) -> (continuation of Deferred@1054101720 after retry RPC@871667701), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c537b23 -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046) -> (continuation of Deferred@1054101720 after retry RPC@871667701))) after lookup
08:03:51.532 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:51.533 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.533 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.533 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.611 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:51.611 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:51.612 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=131), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [2ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [2ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [12ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [14ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [14ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [30ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [31ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [32ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [32ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [48ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [52ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [53ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [53ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [131ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1728297252(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@52198e3b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046) -> (continuation of Deferred@1054101720 after retry RPC@871667701) -> (continuation of Deferred@1238604816 after retry RPC@1586487776), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c537b23 -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046) -> (continuation of Deferred@1054101720 after retry RPC@871667701) -> (continuation of Deferred@1238604816 after retry RPC@1586487776))) after lookup
08:03:51.613 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:51.614 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.614 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.614 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.911 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:51.911 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:51.912 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=432), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [2ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [2ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [12ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [14ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [14ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [30ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [31ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [32ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [32ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [48ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [52ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [53ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [53ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [131ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [132ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [134ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [134ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [428ms] refreshing cache from master, [428ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [431ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1728297252(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@52198e3b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046) -> (continuation of Deferred@1054101720 after retry RPC@871667701) -> (continuation of Deferred@1238604816 after retry RPC@1586487776) -> (continuation of Deferred@429127564 after retry RPC@942835135), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c537b23 -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046) -> (continuation of Deferred@1054101720 after retry RPC@871667701) -> (continuation of Deferred@1238604816 after retry RPC@1586487776) -> (continuation of Deferred@429127564 after retry RPC@942835135))) after lookup
08:03:51.912 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:51.913 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.914 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:51.914 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:52.031 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:52.031 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)[L],3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)]}]
08:03:52.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=551), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [2ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [2ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [12ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [14ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [14ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [30ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [31ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [32ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [32ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [48ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [52ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [53ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [53ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [131ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [132ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [134ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [134ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [428ms] refreshing cache from master, [428ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [431ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [432ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [434ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [434ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [548ms] refreshing cache from master, [548ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [551ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1728297252(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@52198e3b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046) -> (continuation of Deferred@1054101720 after retry RPC@871667701) -> (continuation of Deferred@1238604816 after retry RPC@1586487776) -> (continuation of Deferred@429127564 after retry RPC@942835135) -> (continuation of Deferred@676075416 after retry RPC@75460691), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c537b23 -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046) -> (continuation of Deferred@1054101720 after retry RPC@871667701) -> (continuation of Deferred@1238604816 after retry RPC@1586487776) -> (continuation of Deferred@429127564 after retry RPC@942835135) -> (continuation of Deferred@676075416 after retry RPC@75460691))) after lookup
08:03:52.032 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] connecting to peer
08:03:52.033 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:52.034 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] cleaning up while in state CONNECTING due to: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:52.034 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953) for tablet da718a1094e44290ad9bb1c8d3f5b8f0: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953
08:03:52.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.807811 19469 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 11 FOLLOWER]: Starting pre-election (detected failure of leader 13ed5f47b5c14458a6cc3efb8918cd2c)
08:03:52.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.807951 19469 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 11 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:52.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.808463 19469 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 12 pre-election: Requested pre-vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:52.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:52.809689 19166 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 12 pre-election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:52.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.813190 19410 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b" is_pre_election: true
08:03:52.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.813516 19410 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3782bdbeb06c497ca787a0b7b2e6c918 in term 10.
08:03:52.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.814221 19165 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 12 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:52.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.814394 19469 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 11 FOLLOWER]: Leader pre-election won for term 12
08:03:52.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.814458 19469 raft_consensus.cc:493] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 11 FOLLOWER]: Starting leader election (detected failure of leader 13ed5f47b5c14458a6cc3efb8918cd2c)
08:03:52.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.814507 19469 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 11 FOLLOWER]: Advancing to term 12
08:03:52.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.815281 19469 raft_consensus.cc:515] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 12 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:52.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.815596 19469 leader_election.cc:290] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 12 election: Requested vote from peers 010d2db890fc482bb61ce7809297487b (127.16.186.67:41083), 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:52.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.815919 19410 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "da718a1094e44290ad9bb1c8d3f5b8f0" candidate_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "010d2db890fc482bb61ce7809297487b"
08:03:52.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.816035 19410 raft_consensus.cc:3060] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 10 FOLLOWER]: Advancing to term 12
08:03:52.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:52.816411 19166 leader_election.cc:336] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 12 election: RPC error from VoteRequest() call to peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111)
08:03:52.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.817780 19410 raft_consensus.cc:2468] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 12 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3782bdbeb06c497ca787a0b7b2e6c918 in term 12.
08:03:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.818054 19165 leader_election.cc:304] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [CANDIDATE]: Term 12 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010d2db890fc482bb61ce7809297487b, 3782bdbeb06c497ca787a0b7b2e6c918; no voters: 13ed5f47b5c14458a6cc3efb8918cd2c
08:03:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.818208 19469 raft_consensus.cc:2804] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 12 FOLLOWER]: Leader election won for term 12
08:03:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.818420 19469 raft_consensus.cc:697] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [term 12 LEADER]: Becoming Leader. State: Replica: 3782bdbeb06c497ca787a0b7b2e6c918, State: Running, Role: LEADER
08:03:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.818722 19469 consensus_queue.cc:237] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 12, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:52.821230 17171 catalog_manager.cc:5696] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 reported cstate change: term changed from 11 to 12, leader changed from 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65) to 3782bdbeb06c497ca787a0b7b2e6c918 (127.16.186.66). New cstate: current_term: 12 leader_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } health_report { overall_health: UNKNOWN } } }
08:03:53.111 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet da718a1094e44290ad9bb1c8d3f5b8f0 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455' with partition [<start>, <end>)
08:03:53.111 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=da718a1094e44290ad9bb1c8d3f5b8f0@[010d2db890fc482bb61ce7809297487b(127.16.186.67:41083),13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953),3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)[L]]}]
08:03:53.112 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1631), Traces: [0ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [2ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [2ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [12ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [14ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [14ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [30ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [31ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [32ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [32ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [48ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [52ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [53ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [53ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [131ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [132ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [134ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [134ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [428ms] refreshing cache from master, [428ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [431ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [432ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [434ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [434ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [548ms] refreshing cache from master, [548ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [551ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, [552ms] sending RPC to server 13ed5f47b5c14458a6cc3efb8918cd2c, [554ms] received response from server 13ed5f47b5c14458a6cc3efb8918cd2c: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [554ms] delaying RPC due to: Network error: Failed to connect to peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953): Connection refused: /127.16.186.65:40953, [1628ms] refreshing cache from master, [1628ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.186.126:40625, [1631ms] Sub RPC GetTableLocations: received response from server master-127.16.186.126:40625: OK, deferred=Deferred@1728297252(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7758336730@52198e3b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046) -> (continuation of Deferred@1054101720 after retry RPC@871667701) -> (continuation of Deferred@1238604816 after retry RPC@1586487776) -> (continuation of Deferred@429127564 after retry RPC@942835135) -> (continuation of Deferred@676075416 after retry RPC@75460691) -> (continuation of Deferred@1424565392 after retry RPC@736836534), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c537b23 -> wakeup thread Time-limited test -> (continuation of Deferred@262592610 after retry RPC@1492857046) -> (continuation of Deferred@1054101720 after retry RPC@871667701) -> (continuation of Deferred@1238604816 after retry RPC@1586487776) -> (continuation of Deferred@429127564 after retry RPC@942835135) -> (continuation of Deferred@676075416 after retry RPC@75460691) -> (continuation of Deferred@1424565392 after retry RPC@736836534))) after lookup
08:03:53.112 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] connecting to peer
08:03:53.113 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] Successfully connected to peer
08:03:53.126 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0x6125ef8e, L:/127.0.0.1:37770 - R:/127.16.186.66:46109] using TOKEN/null
08:03:53.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.130224 19410 raft_consensus.cc:1275] T da718a1094e44290ad9bb1c8d3f5b8f0 P 010d2db890fc482bb61ce7809297487b [term 12 FOLLOWER]: Refusing update from remote peer 3782bdbeb06c497ca787a0b7b2e6c918: Log matching property violated. Preceding OpId in replica: term: 10 index: 36. Preceding OpId from leader: term: 12 index: 42. (index mismatch)
08:03:53.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.130434 19166 consensus_peers.cc:597] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 -> Peer 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953): Couldn't send request to peer 13ed5f47b5c14458a6cc3efb8918cd2c. Status: Network error: Client connection negotiation failed: client connection to 127.16.186.65:40953: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.130858 19474 consensus_queue.cc:1048] T da718a1094e44290ad9bb1c8d3f5b8f0 P 3782bdbeb06c497ca787a0b7b2e6c918 [LEADER]: Connected to new peer: Peer: permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
08:03:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.137151 19487 mvcc.cc:204] Tried to move back new op lower bound from 7287158510095171584 to 7287158508828364800. Current Snapshot: MvccSnapshot[applied={T|T < 7287158510095171584}]
08:03:53.148 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.186.65:40953
08:03:53.149 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

08:03:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.149612 17129 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu
08:03:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data
08:03:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/logs
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.65:40953
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.65
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39271
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.65
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.246516 19496 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.246706 19496 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.246747 19496 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:53.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.249925 19496 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:53.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.250027 19496 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.186.65
08:03:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.253623 19496 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.186.84:44425
08:03:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data
08:03:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.186.65:40953
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.186.65
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39271
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.186.126:40625,127.16.186.125:39845,127.16.186.124:44471
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.19496
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.186.65
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/logs
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:03:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 07:43:18 UTC on bdcb31816ec0
08:03:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11952
08:03:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.254652 19496 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:53.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.256120 19496 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.263674 19502 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:53.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.263698 19501 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:53.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 08:03:53.263835 19504 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:53.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.264040 19496 server_base.cc:1061] running on GCE node
08:03:53.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.264631 19496 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.265141 19496 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.266280 19496 hybrid_clock.cc:648] HybridClock initialized: now 1779091433266267 us; error 34 us; skew 500 ppm
08:03:53.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.267947 19496 webserver.cc:533] Webserver started at http://127.16.186.65:39271/ using document root <none> and password file <none>
08:03:53.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.268486 19496 fs_manager.cc:362] Metadata directory not provided
08:03:53.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.268543 19496 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:53.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.271729 19496 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.000s	sys 0.001s
08:03:53.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.273433 19510 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:03:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.274359 19496 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.001s
08:03:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.274456 19496 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data,/tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13ed5f47b5c14458a6cc3efb8918cd2c"
08:03:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 08:03:21 on dist-test-slave-lw51"
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.274819 19496 fs_report.cc:389] FS layout report
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/wal
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/data
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:53.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.297807 19496 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:03:53.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.298471 19496 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:53.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.298638 19496 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:53.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.299201 19496 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:53.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.300484 19517 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
08:03:53.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.303660 19496 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
08:03:53.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.303715 19496 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
08:03:53.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.303773 19496 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
08:03:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.305195 19496 ts_tablet_manager.cc:616] Registered 1 tablets
08:03:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.305238 19496 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
08:03:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.305596 19517 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap starting.
08:03:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.315435 19496 rpc_server.cc:307] RPC server started. Bound to: 127.16.186.65:40953
08:03:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.315506 19624 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.186.65:40953 every 8 connection(s)
08:03:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.316474 19496 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskrHGDhh/test-tmp/mini-kudu-cluster13681230878684276663/ts-0/data/info.pb
08:03:53.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.323096 19517 log.cc:826] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Log is configured to *not* fsync() on all Append() calls
08:03:53.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.324373 17129 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu as pid 19496
08:03:53.324 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:03:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.326041 19625 heartbeater.cc:344] Connected to a master server at 127.16.186.124:44471
08:03:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.326278 19625 heartbeater.cc:461] Registering TS with master...
08:03:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.326828 19625 heartbeater.cc:507] Master 127.16.186.124:44471 requested a full tablet report, sending...
08:03:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.328810 19626 heartbeater.cc:344] Connected to a master server at 127.16.186.126:40625
08:03:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.328888 19626 heartbeater.cc:461] Registering TS with master...
08:03:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.328970 17313 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.329077 19626 heartbeater.cc:507] Master 127.16.186.126:40625 requested a full tablet report, sending...
08:03:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.329847 17171 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.330704 17171 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.186.65:45051
08:03:53.332 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:53.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.333590 19627 heartbeater.cc:344] Connected to a master server at 127.16.186.125:39845
08:03:53.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.333674 19627 heartbeater.cc:461] Registering TS with master...
08:03:53.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.334579 19627 heartbeater.cc:507] Master 127.16.186.125:39845 requested a full tablet report, sending...
08:03:53.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.335448 17243 ts_manager.cc:194] Re-registered known tserver with Master: 13ed5f47b5c14458a6cc3efb8918cd2c (127.16.186.65:40953)
08:03:53.337 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779091402455, tablet=da718a1094e44290ad9bb1c8d3f5b8f0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
08:03:53.342 [DEBUG - main] (Connection.java:575) [peer master-127.16.186.124:44471(127.16.186.124:44471)] disconnecting while in state READY
08:03:53.343 [DEBUG - main] (Connection.java:575) [peer master-127.16.186.125:39845(127.16.186.125:39845)] disconnecting while in state READY
08:03:53.343 [DEBUG - main] (Connection.java:575) [peer 13ed5f47b5c14458a6cc3efb8918cd2c(127.16.186.65:40953)] disconnecting while in state TERMINATED
08:03:53.344 [DEBUG - main] (Connection.java:575) [peer master-127.16.186.126:40625(127.16.186.126:40625)] disconnecting while in state READY
08:03:53.344 [DEBUG - main] (Connection.java:575) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] disconnecting while in state READY
08:03:53.344 [DEBUG - main] (Connection.java:575) [peer 010d2db890fc482bb61ce7809297487b(127.16.186.67:41083)] disconnecting while in state TERMINATED
08:03:53.346 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.186.124:44471(127.16.186.124:44471)] handling channelInactive
08:03:53.346 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.186.126:40625(127.16.186.126:40625)] handling channelInactive
08:03:53.346 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.186.124:44471(127.16.186.124:44471)] cleaning up while in state READY due to: connection closed
08:03:53.346 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.186.126:40625(127.16.186.126:40625)] cleaning up while in state READY due to: connection closed
08:03:53.347 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] handling channelInactive
08:03:53.347 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 3782bdbeb06c497ca787a0b7b2e6c918(127.16.186.66:46109)] cleaning up while in state READY due to: connection closed
08:03:53.347 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.16.186.125:39845(127.16.186.125:39845)] handling channelInactive
08:03:53.347 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.16.186.125:39845(127.16.186.125:39845)] cleaning up while in state READY due to: connection closed
08:03:53.347 [DEBUG - kudu-nio-4] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:03:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.351125 19517 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.351523 19517 tablet_bootstrap.cc:492] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Bootstrap complete.
08:03:53.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.352712 19517 ts_tablet_manager.cc:1403] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c: Time spent bootstrapping tablet: real 0.047s	user 0.038s	sys 0.003s
08:03:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.355391 19517 raft_consensus.cc:359] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 11 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.355818 19517 raft_consensus.cc:740] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [term 11 FOLLOWER]: Becoming Follower/Learner. State: Replica: 13ed5f47b5c14458a6cc3efb8918cd2c, State: Initialized, Role: FOLLOWER
08:03:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.356338 19517 consensus_queue.cc:260] T da718a1094e44290ad9bb1c8d3f5b8f0 P 13ed5f47b5c14458a6cc3efb8918cd2c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "010d2db890fc482bb61ce7809297487b" member_type: VOTER last_known_addr { host: "127.16.186.67" port: 41083 } } peers { permanent_uuid: "3782bdbeb06c497ca787a0b7b2e6c918" member_type: VOTER last_known_addr { host: "127.16.186.66" port: 46109 } } peers { permanent_uuid: "13ed5f47b5c14458a6cc3efb8918cd2c" member_type: VOTER last_known_addr { host: "127.16.186.65" port: 40953 } }
08:03:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.356719 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 19496
08:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.363605 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 19148
08:03:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.369364 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 19327
08:03:53.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.373994 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 17140
08:03:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.378767 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 17209
08:03:53.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 08:03:53.383313 17129 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrHGDhh/build/debug/bin/kudu with pid 17281
08:03:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T08:03:53Z chronyd exiting

Time: 33.57

OK (1 test)

