JUnit version 4.13.2
.18:29:31.914 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783535371906
18:29:31.980 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
18:29:32.022 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
18:29:32.034 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task5ZdiiQ/test-tmp
18:29:32.062 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
18:29:32.065 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task5ZdiiQ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
18:29:32.252 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

18:29:32.458 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
18:29:32.460 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

18:29:32.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:32.464188  1089 mini_oidc.cc:150] Starting JWKS server
18:29:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:32.467880  1089 webserver.cc:534] Webserver started at http://127.0.0.1:46699/ using document root <none> and password file <none>
18:29:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:32.468487  1089 mini_oidc.cc:186] Starting OIDC Discovery server
18:29:32.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:32.470890  1089 webserver.cc:534] Webserver started at http://127.0.0.1:44147/ using document root <none> and password file <none>
18:29:32.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T18:29:32Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
18:29:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T18:29:32Z Disabled control of system clock
18:29:32.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:32.532768  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:32.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:32.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/wal
18:29:32.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/data
18:29:32.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:29:32.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:29:32.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:29:32.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:32.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:32.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:32.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/logs
18:29:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/data/info.pb
18:29:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:29:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:29:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:29:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:29:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:29:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:29:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:29:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:29:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.126:39121
18:29:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.126
18:29:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:29:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:29:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46699/jwks.json with env {}
18:29:32.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.831012  1101 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:29:32.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.831900  1101 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:29:32.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.832456  1101 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:29:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.865957  1101 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
18:29:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.866250  1101 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:29:32.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.866460  1101 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
18:29:32.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.866652  1101 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
18:29:32.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.866833  1101 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
18:29:32.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.866973  1101 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46699/jwks.json
18:29:32.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:32.903034  1101 master_runner.cc:387] Master server non-default flags:
18:29:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/data
18:29:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/wal
18:29:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:29:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:29:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.126:39121
18:29:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:29:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:29:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:29:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46699/jwks.json
18:29:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/data/info.pb
18:29:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.126
18:29:32.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:32.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:29:32.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:32.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:29:32.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:29:32.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:32.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/logs
18:29:32.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:32.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:29:32.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:29:32.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:29:32.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:29:32.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:29:32.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:29:32.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:29:32.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:29:32.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:29:32.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:32.904590  1101 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:29:32.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:32.906457  1101 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:29:32.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.918489  1107 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:34.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:34.322587  1106 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 1101
18:29:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:34.786144  1101 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.867s	user 0.624s	sys 1.242s
18:29:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:34.786530  1101 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.868s	user 0.624s	sys 1.242s
18:29:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:32.918973  1108 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:34.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:34.787748  1109 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1866 milliseconds
18:29:34.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:34.788743  1110 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:34.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.788697  1101 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:29:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.792856  1101 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:29:34.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.795506  1101 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:29:34.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.796932  1101 hybrid_clock.cc:648] HybridClock initialized: now 1783535374796920 us; error 63 us; skew 500 ppm
18:29:34.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.797799  1101 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:29:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.804611  1101 webserver.cc:534] Webserver started at http://127.1.16.126:41273/ using document root <none> and password file <none>
18:29:34.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.805603  1101 fs_manager.cc:362] Metadata directory not provided
18:29:34.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.805792  1101 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:29:34.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.806264  1101 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:29:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.810979  1101 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/data/instance:
18:29:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fcdf4780f58c451a9bca5afc75f1ec79"
18:29:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:34 on dist-test-slave-l44q"
18:29:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.812299  1101 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/wal/instance:
18:29:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fcdf4780f58c451a9bca5afc75f1ec79"
18:29:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:34 on dist-test-slave-l44q"
18:29:34.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.820080  1101 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.001s
18:29:34.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.826246  1117 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:29:34.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.827558  1101 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
18:29:34.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.827898  1101 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/wal
18:29:34.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fcdf4780f58c451a9bca5afc75f1ec79"
18:29:34.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:34 on dist-test-slave-l44q"
18:29:34.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.828308  1101 fs_report.cc:389] FS layout report
18:29:34.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:29:34.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/wal
18:29:34.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/wal
18:29:34.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/data/data
18:29:34.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:29:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:29:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:29:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:29:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:29:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:29:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:29:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:29:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:29:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:29:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:29:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.891057  1101 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:29:34.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.892525  1101 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:29:34.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.892930  1101 kserver.cc:163] Server-wide thread pool size limit: 3276
18:29:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.966200  1101 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.126:39121
18:29:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.966310  1168 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.126:39121 every 8 connection(s)
18:29:34.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.969177  1101 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/data/info.pb
18:29:34.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.975512  1169 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:29:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.979285  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 1101
18:29:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.979794  1089 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-0/wal/instance
18:29:34.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.984985  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:34.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:34.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/wal
18:29:34.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/data
18:29:34.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:29:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:29:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:29:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/logs
18:29:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/data/info.pb
18:29:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:29:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:29:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:29:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:29:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:34.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:34.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:29:34.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:29:34.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:29:34.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:29:34.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.125:33673
18:29:34.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.125
18:29:34.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:34.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:34.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:34.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:34.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:34.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:29:34.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:34.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:29:34.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46699/jwks.json with env {}
18:29:34.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:34.991073  1169 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } has no permanent_uuid. Determining permanent_uuid...
18:29:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:35.021893  1169 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } has no permanent_uuid. Determining permanent_uuid...
18:29:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.024545  1119 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.125:33673: connect: Connection refused (error 111)
18:29:35.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.029356  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.125:33673: Network error: Client connection negotiation failed: client connection to 127.1.16.125:33673: connect: Connection refused (error 111)
18:29:35.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:35.078872  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } attempt: 1
18:29:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.084296  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.125:33673: Network error: Client connection negotiation failed: client connection to 127.1.16.125:33673: connect: Connection refused (error 111)
18:29:35.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:35.152729  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } attempt: 2
18:29:35.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.157498  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.125:33673: Network error: Client connection negotiation failed: client connection to 127.1.16.125:33673: connect: Connection refused (error 111)
18:29:35.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:35.248883  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } attempt: 3
18:29:35.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.253722  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.125:33673: Network error: Client connection negotiation failed: client connection to 127.1.16.125:33673: connect: Connection refused (error 111)
18:29:35.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.308020  1171 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:29:35.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.308773  1171 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:29:35.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.309507  1171 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:29:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.341559  1171 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
18:29:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.341933  1171 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:29:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.342262  1171 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
18:29:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.342628  1171 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
18:29:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.343006  1171 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
18:29:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.343288  1171 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46699/jwks.json
18:29:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:35.379000  1171 master_runner.cc:387] Master server non-default flags:
18:29:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:35.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:35.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:35.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:35.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/data
18:29:35.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/wal
18:29:35.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:29:35.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:35.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:35.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:35.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:29:35.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.125:33673
18:29:35.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:29:35.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:29:35.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:29:35.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46699/jwks.json
18:29:35.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:35.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:35.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/data/info.pb
18:29:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.125
18:29:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:29:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:29:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:29:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/logs
18:29:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:29:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:29:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:29:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:29:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:29:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:29:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:29:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:29:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:29:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:35.380687  1171 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:29:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:35.382946  1171 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:29:35.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.394827  1180 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:35.397153  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } attempt: 4
18:29:35.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.403472  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.125:33673: Network error: Client connection negotiation failed: client connection to 127.1.16.125:33673: connect: Connection refused (error 111)
18:29:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:35.703016  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } attempt: 5
18:29:35.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.708958  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.125:33673: Network error: Client connection negotiation failed: client connection to 127.1.16.125:33673: connect: Connection refused (error 111)
18:29:36.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.256407  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } attempt: 6
18:29:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:36.263522  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.125:33673: Network error: Client connection negotiation failed: client connection to 127.1.16.125:33673: connect: Connection refused (error 111)
18:29:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:35.395382  1181 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:36.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:36.610567  1182 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
18:29:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:36.612433  1183 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.612435  1171 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:29:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.615161  1171 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:29:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.617842  1171 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:29:36.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.619225  1171 hybrid_clock.cc:648] HybridClock initialized: now 1783535376619187 us; error 35 us; skew 500 ppm
18:29:36.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.620157  1171 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:29:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.626323  1171 webserver.cc:534] Webserver started at http://127.1.16.125:37523/ using document root <none> and password file <none>
18:29:36.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.627324  1171 fs_manager.cc:362] Metadata directory not provided
18:29:36.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.627573  1171 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:29:36.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.628158  1171 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:29:36.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.632524  1171 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/data/instance:
18:29:36.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d08e2c390020456fa63602c5fa0d773f"
18:29:36.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:36 on dist-test-slave-l44q"
18:29:36.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.633864  1171 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/wal/instance:
18:29:36.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d08e2c390020456fa63602c5fa0d773f"
18:29:36.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:36 on dist-test-slave-l44q"
18:29:36.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.641040  1171 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.001s
18:29:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.646939  1192 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:29:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.648192  1171 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.002s
18:29:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.648658  1171 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/wal
18:29:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d08e2c390020456fa63602c5fa0d773f"
18:29:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:36 on dist-test-slave-l44q"
18:29:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.649051  1171 fs_report.cc:389] FS layout report
18:29:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:29:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/wal
18:29:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/wal
18:29:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/data/data
18:29:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:29:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:29:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:29:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:29:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:29:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:29:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:29:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:29:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:29:36.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:29:36.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:29:36.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.712391  1171 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:29:36.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.713862  1171 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:29:36.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.714272  1171 kserver.cc:163] Server-wide thread pool size limit: 3276
18:29:36.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.789880  1171 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.125:33673
18:29:36.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.789973  1243 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.125:33673 every 8 connection(s)
18:29:36.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.792865  1171 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/data/info.pb
18:29:36.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.793779  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 1171
18:29:36.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.794245  1089 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-1/wal/instance
18:29:36.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.797129  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/wal
18:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/data
18:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/logs
18:29:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/data/info.pb
18:29:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:29:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:29:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:29:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:29:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:29:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:29:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:29:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:29:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.124:45813
18:29:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.124
18:29:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:29:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:29:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46699/jwks.json with env {}
18:29:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.802170  1244 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:29:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.815587  1244 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } has no permanent_uuid. Determining permanent_uuid...
18:29:36.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.837004  1244 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } has no permanent_uuid. Determining permanent_uuid...
18:29:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.851981  1244 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } has no permanent_uuid. Determining permanent_uuid...
18:29:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:36.854664  1196 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:36.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:36.859685  1244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:36.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.909289  1244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 1
18:29:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:36.914678  1244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:36.983112  1244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 2
18:29:36.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:36.989099  1244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.080520  1244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 3
18:29:37.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.084841  1244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:37.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.137156  1246 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:29:37.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.137730  1246 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:29:37.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.138167  1246 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:29:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.173818  1246 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
18:29:37.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.174095  1246 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:29:37.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.174511  1246 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
18:29:37.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.174954  1246 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
18:29:37.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.175186  1246 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
18:29:37.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.175347  1246 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46699/jwks.json
18:29:37.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.212790  1246 master_runner.cc:387] Master server non-default flags:
18:29:37.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:37.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:37.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:37.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:37.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/data
18:29:37.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/wal
18:29:37.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.124:45813
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46699/jwks.json
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/data/info.pb
18:29:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.124
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/logs
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:29:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:29:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:29:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:29:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:29:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:29:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:29:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.214043  1246 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:29:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.216104  1246 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:29:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.228391  1244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 4
18:29:37.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.229082  1256 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.235214  1244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:37.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.324188  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } attempt: 7
18:29:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.349021  1169 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } has no permanent_uuid. Determining permanent_uuid...
18:29:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.360157  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:37.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.418725  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 1
18:29:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.426182  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.507756  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 2
18:29:37.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.517446  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.534658  1244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 5
18:29:37.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.542445  1244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.603318  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 3
18:29:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.610198  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:37.750833  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 4
18:29:37.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.758128  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.041700  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 5
18:29:38.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:38.048709  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.090000  1244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 6
18:29:38.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:38.101042  1244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:38.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.231627  1257 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:38.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:37.233906  1259 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:38.514225  1258 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1281 milliseconds
18:29:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.514366  1246 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:29:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.518257  1246 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:29:38.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.520787  1246 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:29:38.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.522120  1246 hybrid_clock.cc:648] HybridClock initialized: now 1783535378522104 us; error 61 us; skew 500 ppm
18:29:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.522897  1246 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:29:38.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.528884  1246 webserver.cc:534] Webserver started at http://127.1.16.124:41779/ using document root <none> and password file <none>
18:29:38.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.529752  1246 fs_manager.cc:362] Metadata directory not provided
18:29:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.529932  1246 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:29:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.530309  1246 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:29:38.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.534849  1246 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/data/instance:
18:29:38.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b681ff314a4145938d72f85c074f6baf"
18:29:38.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:38 on dist-test-slave-l44q"
18:29:38.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.535948  1246 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/wal/instance:
18:29:38.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b681ff314a4145938d72f85c074f6baf"
18:29:38.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:38 on dist-test-slave-l44q"
18:29:38.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.543233  1246 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.000s
18:29:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.549547  1270 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:29:38.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.550784  1246 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.004s	sys 0.000s
18:29:38.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.551173  1246 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/wal
18:29:38.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b681ff314a4145938d72f85c074f6baf"
18:29:38.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:38 on dist-test-slave-l44q"
18:29:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.551589  1246 fs_report.cc:389] FS layout report
18:29:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:29:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/wal
18:29:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/wal
18:29:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/data/data
18:29:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:29:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:29:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:29:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:29:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:29:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:29:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:29:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:29:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:29:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:29:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:29:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.601257  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 6
18:29:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.607486  1246 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:29:38.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:38.608242  1169 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.16.124:45813: Network error: Client connection negotiation failed: client connection to 127.1.16.124:45813: connect: Connection refused (error 111)
18:29:38.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.609187  1246 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:29:38.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.609702  1246 kserver.cc:163] Server-wide thread pool size limit: 3276
18:29:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.686285  1246 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.124:45813
18:29:38.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.686368  1323 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.124:45813 every 8 connection(s)
18:29:38.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.689098  1246 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/data/info.pb
18:29:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.694160  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 1246
18:29:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.694696  1089 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/master-2/wal/instance
18:29:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.697335  1324 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:29:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.712111  1324 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } has no permanent_uuid. Determining permanent_uuid...
18:29:38.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.735553  1324 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } has no permanent_uuid. Determining permanent_uuid...
18:29:38.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.746671  1324 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } has no permanent_uuid. Determining permanent_uuid...
18:29:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.762948  1324 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf: Bootstrap starting.
18:29:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.769693  1324 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf: Neither blocks nor log segments found. Creating new log.
18:29:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.771519  1324 log.cc:826] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf: Log is configured to *not* fsync() on all Append() calls
18:29:38.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.776895  1324 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf: No bootstrap required, opened a new log
18:29:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.800905  1324 raft_consensus.cc:359] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } }
18:29:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.801802  1324 raft_consensus.cc:385] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:29:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.802090  1324 raft_consensus.cc:740] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b681ff314a4145938d72f85c074f6baf, State: Initialized, Role: FOLLOWER
18:29:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.803030  1324 consensus_queue.cc:260] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } }
18:29:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.805984  1333 sys_catalog.cc:455] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } } }
18:29:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.806690  1333 sys_catalog.cc:458] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [sys.catalog]: This master's current role is: FOLLOWER
18:29:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.808095  1324 sys_catalog.cc:565] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [sys.catalog]: configured and running, proceeding with master startup.
18:29:38.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:38.842656  1344 catalog_manager.cc:1593] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:29:38.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:38.843605  1344 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:29:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.837371  1246 master_runner.cc:187] Error getting master registration for 127.1.16.126:39121: OK, instance_id { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" instance_seqno: 1783535374946288 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:29:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.858227  1246 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
18:29:38.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:38.865701  1278 tablet.cc:2406] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:29:38.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.885790  1333 raft_consensus.cc:493] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:29:38.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.886409  1333 raft_consensus.cc:515] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } }
18:29:38.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:38.889602  1333 leader_election.cc:290] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fcdf4780f58c451a9bca5afc75f1ec79 (127.1.16.126:39121), d08e2c390020456fa63602c5fa0d773f (127.1.16.125:33673)
18:29:39.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.161614  1244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 7
18:29:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.184619  1244 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f: Bootstrap starting.
18:29:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.193754  1244 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f: Neither blocks nor log segments found. Creating new log.
18:29:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.196211  1244 log.cc:826] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f: Log is configured to *not* fsync() on all Append() calls
18:29:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.201766  1244 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f: No bootstrap required, opened a new log
18:29:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.235946  1244 raft_consensus.cc:359] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } }
18:29:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.236896  1244 raft_consensus.cc:385] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:29:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.237221  1244 raft_consensus.cc:740] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d08e2c390020456fa63602c5fa0d773f, State: Initialized, Role: FOLLOWER
18:29:39.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.239563  1244 consensus_queue.cc:260] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } }
18:29:39.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.243832  1348 sys_catalog.cc:455] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } } }
18:29:39.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.244714  1348 sys_catalog.cc:458] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [sys.catalog]: This master's current role is: FOLLOWER
18:29:39.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.245970  1244 sys_catalog.cc:565] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [sys.catalog]: configured and running, proceeding with master startup.
18:29:39.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.246088  1219 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b681ff314a4145938d72f85c074f6baf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d08e2c390020456fa63602c5fa0d773f" is_pre_election: true
18:29:39.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.246897  1219 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b681ff314a4145938d72f85c074f6baf in term 0.
18:29:39.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.248500  1272 leader_election.cc:304] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b681ff314a4145938d72f85c074f6baf, d08e2c390020456fa63602c5fa0d773f; no voters: 
18:29:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.249513  1333 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 0 FOLLOWER]: Leader pre-election won for term 1
18:29:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.249936  1333 raft_consensus.cc:493] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:29:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.250260  1333 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 0 FOLLOWER]: Advancing to term 1
18:29:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.258992  1333 raft_consensus.cc:515] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } }
18:29:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.261914  1333 leader_election.cc:290] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [CANDIDATE]: Term 1 election: Requested vote from peers fcdf4780f58c451a9bca5afc75f1ec79 (127.1.16.126:39121), d08e2c390020456fa63602c5fa0d773f (127.1.16.125:33673)
18:29:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.262145  1218 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b681ff314a4145938d72f85c074f6baf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d08e2c390020456fa63602c5fa0d773f"
18:29:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.262758  1218 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [term 0 FOLLOWER]: Advancing to term 1
18:29:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.269637  1218 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b681ff314a4145938d72f85c074f6baf in term 1.
18:29:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.271008  1272 leader_election.cc:304] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b681ff314a4145938d72f85c074f6baf, d08e2c390020456fa63602c5fa0d773f; no voters: 
18:29:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.274535  1333 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 1 FOLLOWER]: Leader election won for term 1
18:29:39.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.280189  1333 raft_consensus.cc:697] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [term 1 LEADER]: Becoming Leader. State: Replica: b681ff314a4145938d72f85c074f6baf, State: Running, Role: LEADER
18:29:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.281500  1333 consensus_queue.cc:237] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } }
18:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.297348  1357 sys_catalog.cc:455] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [sys.catalog]: SysCatalogTable state changed. Reason: New leader b681ff314a4145938d72f85c074f6baf. Latest consensus state: current_term: 1 leader_uuid: "b681ff314a4145938d72f85c074f6baf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } } }
18:29:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.298456  1357 sys_catalog.cc:458] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [sys.catalog]: This master's current role is: LEADER
18:29:39.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.304971  1361 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
18:29:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.312734  1361 catalog_manager.cc:1519] Initializing Kudu cluster ID...
18:29:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.341545  1218 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [term 1 FOLLOWER]: Refusing update from remote peer b681ff314a4145938d72f85c074f6baf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:29:39.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.343955  1357 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:29:39.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:39.346984  1362 catalog_manager.cc:1593] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:29:39.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:39.347783  1362 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:29:39.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.309580  1171 master_runner.cc:187] Error getting master registration for 127.1.16.126:39121: OK, instance_id { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" instance_seqno: 1783535374946288 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:29:39.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.369988  1171 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
18:29:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.384708  1348 sys_catalog.cc:455] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [sys.catalog]: SysCatalogTable state changed. Reason: New leader b681ff314a4145938d72f85c074f6baf. Latest consensus state: current_term: 1 leader_uuid: "b681ff314a4145938d72f85c074f6baf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } } }
18:29:39.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.389746  1348 sys_catalog.cc:458] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [sys.catalog]: This master's current role is: FOLLOWER
18:29:39.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.396703  1365 mvcc.cc:204] Tried to move back new op lower bound from 7305360913764294656 to 7305360913549316096. Current Snapshot: MvccSnapshot[applied={T|T < 7305360913764294656}]
18:29:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.405254  1333 sys_catalog.cc:455] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b681ff314a4145938d72f85c074f6baf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } } }
18:29:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.405962  1333 sys_catalog.cc:458] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [sys.catalog]: This master's current role is: LEADER
18:29:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.407588  1361 catalog_manager.cc:1382] Generated new cluster ID: 52648f3d0eab4b8d9fc43ce60a3337da
18:29:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.407863  1361 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
18:29:39.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.408310  1348 sys_catalog.cc:455] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b681ff314a4145938d72f85c074f6baf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } } }
18:29:39.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.409009  1348 sys_catalog.cc:458] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f [sys.catalog]: This master's current role is: FOLLOWER
18:29:39.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.440594  1361 catalog_manager.cc:1405] Generated new certificate authority record
18:29:39.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.442690  1361 catalog_manager.cc:1539] Loading token signing keys...
18:29:39.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.478833  1361 catalog_manager.cc:6069] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf: Generated new TSK 0
18:29:39.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.480417  1361 catalog_manager.cc:1549] Initializing in-progress tserver states...
18:29:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.508080  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:29:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data
18:29:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:29:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:29:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:29:39.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:39.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:39.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:39.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/logs
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:29:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.65:0
18:29:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.65
18:29:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.65
18:29:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:29:39.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.641645  1169 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } attempt: 7
18:29:39.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.705232  1169 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79: Bootstrap starting.
18:29:39.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.734958  1169 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79: Neither blocks nor log segments found. Creating new log.
18:29:39.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.742547  1169 log.cc:826] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79: Log is configured to *not* fsync() on all Append() calls
18:29:39.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.777292  1169 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79: No bootstrap required, opened a new log
18:29:39.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.934302  1169 raft_consensus.cc:359] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } }
18:29:39.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.940163  1169 raft_consensus.cc:385] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:29:39.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.942334  1169 raft_consensus.cc:740] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fcdf4780f58c451a9bca5afc75f1ec79, State: Initialized, Role: FOLLOWER
18:29:39.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.948760  1169 consensus_queue.cc:260] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } }
18:29:39.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.989902  1144 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b681ff314a4145938d72f85c074f6baf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" is_pre_election: true
18:29:39.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.990760  1144 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b681ff314a4145938d72f85c074f6baf in term 0.
18:29:39.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.992841  1142 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [term 0 FOLLOWER]: Advancing to term 1
18:29:39.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.997082  1143 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b681ff314a4145938d72f85c074f6baf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fcdf4780f58c451a9bca5afc75f1ec79"
18:29:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.991305  1144 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:35030 (request call id 2) took 1101 ms. Trace:
18:29:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:39.998741  1144 rpcz_store.cc:276] 0708 18:29:38.889461 (+     0us) service_pool.cc:167] Inserting onto call queue
18:29:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:29:38.889632 (+   171us) service_pool.cc:224] Handling call
18:29:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:29:39.991248 (+1101616us) inbound_call.cc:177] Queueing success response
18:29:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
18:29:40.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.004488  1142 raft_consensus.cc:1275] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [term 1 FOLLOWER]: Refusing update from remote peer b681ff314a4145938d72f85c074f6baf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:29:40.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.006320  1333 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:29:40.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.016389  1372 sys_catalog.cc:455] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } } }
18:29:40.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.018703  1372 sys_catalog.cc:458] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [sys.catalog]: This master's current role is: FOLLOWER
18:29:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.017583  1169 sys_catalog.cc:565] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [sys.catalog]: configured and running, proceeding with master startup.
18:29:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.067840  1374 mvcc.cc:204] Tried to move back new op lower bound from 7305360913764294656 to 7305360913549316096. Current Snapshot: MvccSnapshot[applied={T|T < 7305360913764294656}]
18:29:40.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.103974  1373 sys_catalog.cc:455] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b681ff314a4145938d72f85c074f6baf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } } }
18:29:40.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.104934  1373 sys_catalog.cc:458] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [sys.catalog]: This master's current role is: FOLLOWER
18:29:40.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.103560  1372 sys_catalog.cc:455] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b681ff314a4145938d72f85c074f6baf. Latest consensus state: current_term: 1 leader_uuid: "b681ff314a4145938d72f85c074f6baf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } } }
18:29:40.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.106096  1372 sys_catalog.cc:458] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79 [sys.catalog]: This master's current role is: FOLLOWER
18:29:40.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:40.140421  1369 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:29:40.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:40.141894  1369 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:29:40.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:40.143289  1369 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:29:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.183379  1393 catalog_manager.cc:1294] Loaded cluster ID: 52648f3d0eab4b8d9fc43ce60a3337da
18:29:40.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.183658  1393 catalog_manager.cc:1587] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79: loading cluster ID for follower catalog manager: success
18:29:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.206971  1393 catalog_manager.cc:1609] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79: acquiring CA information for follower catalog manager: success
18:29:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:40.212461  1369 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:29:40.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:40.214197  1369 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.65
18:29:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.230768  1393 catalog_manager.cc:1637] T 00000000000000000000000000000000 P fcdf4780f58c451a9bca5afc75f1ec79: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:29:40.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.285392  1369 tablet_server_runner.cc:78] Tablet server non-default flags:
18:29:40.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:40.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:40.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:40.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:40.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data
18:29:40.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:29:40.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.65:0
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.65
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:29:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.65
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/logs
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:29:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:29:40.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:29:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:29:40.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:29:40.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:29:40.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.303112  1369 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:29:40.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.306196  1369 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:29:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:40.342022  1396 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:40.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.354112  1362 catalog_manager.cc:1294] Loaded cluster ID: 52648f3d0eab4b8d9fc43ce60a3337da
18:29:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.354946  1362 catalog_manager.cc:1587] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f: loading cluster ID for follower catalog manager: success
18:29:40.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.363008  1362 catalog_manager.cc:1609] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f: acquiring CA information for follower catalog manager: success
18:29:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.371097  1362 catalog_manager.cc:1637] T 00000000000000000000000000000000 P d08e2c390020456fa63602c5fa0d773f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:29:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.619293  1401 sys_catalog.cc:455] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b681ff314a4145938d72f85c074f6baf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fcdf4780f58c451a9bca5afc75f1ec79" member_type: VOTER last_known_addr { host: "127.1.16.126" port: 39121 } } peers { permanent_uuid: "d08e2c390020456fa63602c5fa0d773f" member_type: VOTER last_known_addr { host: "127.1.16.125" port: 33673 } } peers { permanent_uuid: "b681ff314a4145938d72f85c074f6baf" member_type: VOTER last_known_addr { host: "127.1.16.124" port: 45813 } } }
18:29:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:40.622700  1401 sys_catalog.cc:458] T 00000000000000000000000000000000 P b681ff314a4145938d72f85c074f6baf [sys.catalog]: This master's current role is: LEADER
18:29:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:41.737004  1395 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 1369
18:29:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:42.015789  1395 kernel_stack_watchdog.cc:198] Thread 1369 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 370ms:
18:29:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
18:29:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
18:29:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
18:29:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
18:29:42.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:40.352831  1397 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:42.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:42.030761  1369 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.667s	user 0.000s	sys 0.007s
18:29:42.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:42.031438  1369 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.667s	user 0.000s	sys 0.007s
18:29:42.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.031893  1369 server_base.cc:1061] running on GCE node
18:29:42.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:42.036290  1399 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.037921  1369 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:29:42.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.047817  1369 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:29:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.055929  1369 hybrid_clock.cc:648] HybridClock initialized: now 1783535382055878 us; error 63 us; skew 500 ppm
18:29:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.057354  1369 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:29:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.077596  1369 webserver.cc:534] Webserver started at http://127.1.16.65:44059/ using document root <none> and password file <none>
18:29:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.079322  1369 fs_manager.cc:362] Metadata directory not provided
18:29:42.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.079849  1369 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:29:42.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.080672  1369 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:29:42.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.088439  1369 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/instance:
18:29:42.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8fdfc4241d34c848f5eb3275d07e1f6"
18:29:42.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:42 on dist-test-slave-l44q"
18:29:42.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.090559  1369 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal/instance:
18:29:42.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8fdfc4241d34c848f5eb3275d07e1f6"
18:29:42.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:42 on dist-test-slave-l44q"
18:29:42.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.107657  1369 fs_manager.cc:696] Time spent creating directory manager: real 0.016s	user 0.009s	sys 0.004s
18:29:42.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.123849  1409 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:29:42.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.125960  1369 fs_manager.cc:730] Time spent opening block manager: real 0.012s	user 0.005s	sys 0.004s
18:29:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.126673  1369 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:29:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8fdfc4241d34c848f5eb3275d07e1f6"
18:29:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:42 on dist-test-slave-l44q"
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.127529  1369 fs_report.cc:389] FS layout report
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/data
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:29:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:29:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:29:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:29:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:29:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:29:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:29:42.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.224166  1369 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:29:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.229379  1369 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:29:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.230484  1369 kserver.cc:163] Server-wide thread pool size limit: 3276
18:29:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.242246  1369 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:29:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.255046  1369 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:29:42.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.255733  1369 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.001s	sys 0.000s
18:29:42.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.256309  1369 ts_tablet_manager.cc:616] Registered 0 tablets
18:29:42.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:42.256773  1369 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:29:43.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.114916  1369 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.65:45335
18:29:43.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.115037  1523 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.65:45335 every 8 connection(s)
18:29:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.118230  1369 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:29:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.122190  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 1369
18:29:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.123040  1089 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal/instance
18:29:43.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.139048  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/logs
18:29:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.66:0
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.66
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.66
18:29:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:29:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.208539  1525 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:29:43.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.208992  1525 heartbeater.cc:461] Registering TS with master...
18:29:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.210321  1525 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:29:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.212555  1524 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:29:43.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.212944  1524 heartbeater.cc:461] Registering TS with master...
18:29:43.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.213783  1524 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:29:43.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.217638  1209 ts_manager.cc:194] Registered new tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:29:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.220649  1289 ts_manager.cc:194] Registered new tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:29:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.223275  1289 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.65:54145
18:29:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.227818  1526 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:29:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.228199  1526 heartbeater.cc:461] Registering TS with master...
18:29:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.229588  1526 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:29:43.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.233707  1134 ts_manager.cc:194] Registered new tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:29:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:43.587796  1531 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:29:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:43.588459  1531 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:29:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:43.589036  1531 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:29:43.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:43.620985  1531 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:29:43.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:43.621937  1531 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.66
18:29:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.657683  1531 tablet_server_runner.cc:78] Tablet server non-default flags:
18:29:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data
18:29:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:29:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:43.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:43.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.66:0
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.66
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.66
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/logs
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:29:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:29:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:29:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:29:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:29:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:29:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:29:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:29:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:29:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.659075  1531 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:29:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:43.660787  1531 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:29:43.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:43.681859  1540 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:44.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:44.238345  1524 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:29:45.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.086285  1539 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 1531
18:29:45.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.198505  1539 kernel_stack_watchdog.cc:198] Thread 1531 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 395ms:
18:29:45.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
18:29:45.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
18:29:45.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
18:29:45.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
18:29:45.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.199615  1531 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.516s	user 0.486s	sys 0.974s
18:29:45.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.200059  1531 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.517s	user 0.486s	sys 0.974s
18:29:45.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:43.685755  1541 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:45.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.202307  1543 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.205562  1542 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1509 milliseconds
18:29:45.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.205566  1531 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:29:45.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.206833  1531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:29:45.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.208847  1531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:29:45.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.210253  1531 hybrid_clock.cc:648] HybridClock initialized: now 1783535385210213 us; error 30 us; skew 500 ppm
18:29:45.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.211236  1531 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:29:45.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.219799  1531 webserver.cc:534] Webserver started at http://127.1.16.66:36159/ using document root <none> and password file <none>
18:29:45.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.220916  1531 fs_manager.cc:362] Metadata directory not provided
18:29:45.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.221184  1531 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:29:45.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.221796  1531 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:29:45.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.227174  1531 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/instance:
18:29:45.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9df350dc58374330a22867bfa403ed8d"
18:29:45.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:45 on dist-test-slave-l44q"
18:29:45.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.228487  1531 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal/instance:
18:29:45.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9df350dc58374330a22867bfa403ed8d"
18:29:45.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:45 on dist-test-slave-l44q"
18:29:45.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.236135  1531 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.000s
18:29:45.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.245299  1554 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:29:45.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.246538  1531 fs_manager.cc:730] Time spent opening block manager: real 0.006s	user 0.008s	sys 0.000s
18:29:45.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.246954  1531 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:29:45.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9df350dc58374330a22867bfa403ed8d"
18:29:45.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:45 on dist-test-slave-l44q"
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.247339  1531 fs_report.cc:389] FS layout report
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/data
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:29:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:29:45.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:29:45.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:29:45.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:29:45.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:29:45.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:29:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.312181  1531 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:29:45.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.313966  1531 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:29:45.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.314564  1531 kserver.cc:163] Server-wide thread pool size limit: 3276
18:29:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.317571  1531 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:29:45.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.322360  1531 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:29:45.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.322609  1531 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:29:45.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.322990  1531 ts_tablet_manager.cc:616] Registered 0 tablets
18:29:45.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.323221  1531 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:29:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.515844  1531 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.66:44881
18:29:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.516144  1667 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.66:44881 every 8 connection(s)
18:29:45.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.518757  1531 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:29:45.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.524240  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 1531
18:29:45.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.524890  1089 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal/instance
18:29:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.541939  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:29:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data
18:29:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/logs
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:29:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.67:0
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.67
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.67
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:29:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.582119  1668 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:29:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.582633  1668 heartbeater.cc:461] Registering TS with master...
18:29:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.583264  1669 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:29:45.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.583737  1669 heartbeater.cc:461] Registering TS with master...
18:29:45.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.585268  1668 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:29:45.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.585950  1669 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:29:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.588948  1672 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:29:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.589282  1672 heartbeater.cc:461] Registering TS with master...
18:29:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.590142  1672 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:29:45.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.594354  1134 ts_manager.cc:194] Registered new tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:29:45.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.595990  1209 ts_manager.cc:194] Registered new tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:29:45.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.598415  1289 ts_manager.cc:194] Registered new tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:29:45.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:45.599846  1289 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.66:53065
18:29:45.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.931267  1677 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:29:45.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.931792  1677 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:29:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.932201  1677 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:29:45.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.967890  1677 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:29:45.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:45.968933  1677 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.67
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:46.006632  1677 tablet_server_runner.cc:78] Tablet server non-default flags:
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.67:0
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.67
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.67
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/logs
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:29:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:29:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:29:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:29:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:29:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:29:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:29:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:29:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:29:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:46.007916  1677 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:29:46.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:46.009509  1677 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:29:46.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:46.024708  1684 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:46.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:46.603849  1668 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:29:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:47.429643  1683 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 1677
18:29:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:46.027777  1685 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:47.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:47.896721  1677 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.869s	user 0.715s	sys 1.127s
18:29:47.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:47.897559  1677 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.870s	user 0.716s	sys 1.128s
18:29:47.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:47.899420  1687 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:47.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:47.902644  1686 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1870 milliseconds
18:29:47.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.902680  1677 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:29:47.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.903918  1677 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:29:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.906064  1677 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:29:47.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.907439  1677 hybrid_clock.cc:648] HybridClock initialized: now 1783535387907406 us; error 31 us; skew 500 ppm
18:29:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.908281  1677 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:29:47.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.914237  1677 webserver.cc:534] Webserver started at http://127.1.16.67:37737/ using document root <none> and password file <none>
18:29:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.915151  1677 fs_manager.cc:362] Metadata directory not provided
18:29:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.915387  1677 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:29:47.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.915899  1677 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:29:47.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.920104  1677 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/instance:
18:29:47.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:29:47.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:47 on dist-test-slave-l44q"
18:29:47.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.921312  1677 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal/instance:
18:29:47.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:29:47.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:47 on dist-test-slave-l44q"
18:29:47.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.928292  1677 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.000s
18:29:47.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.933616  1695 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:29:47.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.934746  1677 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.002s
18:29:47.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.935104  1677 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:29:47.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:29:47.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:47 on dist-test-slave-l44q"
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.935443  1677 fs_report.cc:389] FS layout report
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/data
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:29:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:29:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.993178  1677 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:29:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.994961  1677 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:29:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.995538  1677 kserver.cc:163] Server-wide thread pool size limit: 3276
18:29:47.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:47.998790  1677 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:29:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.003000  1677 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:29:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.003234  1677 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:29:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.003576  1677 ts_tablet_manager.cc:616] Registered 0 tablets
18:29:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.003785  1677 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:29:48.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.289381  1677 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.67:37285
18:29:48.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.289507  1807 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.67:37285 every 8 connection(s)
18:29:48.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.293010  1677 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:29:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.303102  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 1677
18:29:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.303735  1089 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal/instance
18:29:48.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.347926  1809 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:29:48.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.348523  1809 heartbeater.cc:461] Registering TS with master...
18:29:48.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.348663  1808 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:29:48.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.348927  1808 heartbeater.cc:461] Registering TS with master...
18:29:48.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.349960  1809 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:29:48.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.350219  1808 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:29:48.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.353268  1209 ts_manager.cc:194] Registered new tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:29:48.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.353196  1289 ts_manager.cc:194] Registered new tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:29:48.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.355412  1289 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.67:43907
18:29:48.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.363582  1813 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:29:48.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.364007  1813 heartbeater.cc:461] Registering TS with master...
18:29:48.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.364724  1813 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:29:48.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.367369  1134 ts_manager.cc:194] Registered new tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:29:48.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:48.372586  1089 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
18:29:48.387 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
18:29:48.388 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

18:29:48.399 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.1.16.126"
      port: 39121
    }
    bound_http_address {
      host: "127.1.16.126"
      port: 41273
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.1.16.125"
      port: 33673
    }
    bound_http_address {
      host: "127.1.16.125"
      port: 37523
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.1.16.124"
      port: 45813
    }
    bound_http_address {
      host: "127.1.16.124"
      port: 41779
    }
  }
}

18:29:48.403 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

18:29:48.405 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.1.16.65"
      port: 45335
    }
    bound_http_address {
      host: "127.1.16.65"
      port: 44059
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.1.16.66"
      port: 44881
    }
    bound_http_address {
      host: "127.1.16.66"
      port: 36159
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.1.16.67"
      port: 37285
    }
    bound_http_address {
      host: "127.1.16.67"
      port: 37737
    }
  }
}

18:29:48.406 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
18:29:48.604 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
18:29:48.661 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
18:29:48.825 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.16.124:45813(127.1.16.124:45813)] connecting to peer
18:29:48.871 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.16.126:39121(127.1.16.126:39121)] connecting to peer
18:29:48.872 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.16.125:33673(127.1.16.125:33673)] connecting to peer
18:29:48.895 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.16.126:39121(127.1.16.126:39121)] Successfully connected to peer
18:29:48.897 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.16.124:45813(127.1.16.124:45813)] Successfully connected to peer
18:29:48.900 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.1.16.125:33673(127.1.16.125:33673)] Successfully connected to peer
18:29:49.010 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.1.16.125
18:29:49.010 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.1.16.126
18:29:49.010 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.1.16.124
18:29:49.250 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x6b1afafc, L:/127.0.0.1:43162 - R:/127.1.16.125:33673] using SASL/PLAIN
18:29:49.250 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xce7ecfbb, L:/127.0.0.1:42136 - R:/127.1.16.126:39121] using SASL/PLAIN
18:29:49.250 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x58563ade, L:/127.0.0.1:57182 - R:/127.1.16.124:45813] using SASL/PLAIN
18:29:49.293 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
18:29:49.294 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=511), Traces: [0ms] refreshing cache from master, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.16.124:45813, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.16.126:39121, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.16.125:33673, [479ms] Sub RPC ConnectToMaster: received response from server master-127.1.16.126:39121: OK, [479ms] Sub RPC ConnectToMaster: received response from server master-127.1.16.125:33673: OK, [479ms] Sub RPC ConnectToMaster: received response from server master-127.1.16.124:45813: OK, deferred=null) after lookup
18:29:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.350836  1288 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57182:
18:29:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770"
18:29:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
18:29:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
18:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
18:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:29:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
18:29:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:29:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:29:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:29:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:29:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:29:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:29:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:29:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:29:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:29:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:29:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:29:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:29:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:29:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
18:29:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
18:29:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
18:29:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
18:29:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
18:29:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
18:29:49.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:29:49.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
18:29:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.361158  1808 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:29:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:49.363668  1288 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
18:29:49.445 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table ce2a5a51f13446bbab444c10f9739391
18:29:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.493785  1743 tablet_service.cc:1511] Processing CreateTablet for tablet 41dcfd2a13434555b4e5a2b746a3745f (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770 [id=ce2a5a51f13446bbab444c10f9739391]), partition=RANGE (key) PARTITION UNBOUNDED
18:29:49.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.496511  1743 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 41dcfd2a13434555b4e5a2b746a3745f. 1 dirs total, 0 dirs full, 0 dirs failed
18:29:49.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.513586  1603 tablet_service.cc:1511] Processing CreateTablet for tablet 41dcfd2a13434555b4e5a2b746a3745f (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770 [id=ce2a5a51f13446bbab444c10f9739391]), partition=RANGE (key) PARTITION UNBOUNDED
18:29:49.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.516760  1603 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 41dcfd2a13434555b4e5a2b746a3745f. 1 dirs total, 0 dirs full, 0 dirs failed
18:29:49.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.519511  1458 tablet_service.cc:1511] Processing CreateTablet for tablet 41dcfd2a13434555b4e5a2b746a3745f (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770 [id=ce2a5a51f13446bbab444c10f9739391]), partition=RANGE (key) PARTITION UNBOUNDED
18:29:49.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.521353  1458 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 41dcfd2a13434555b4e5a2b746a3745f. 1 dirs total, 0 dirs full, 0 dirs failed
18:29:49.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.549826  1841 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Bootstrap starting.
18:29:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.561072  1843 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap starting.
18:29:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.561251  1842 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Bootstrap starting.
18:29:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.570695  1842 tablet_bootstrap.cc:654] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Neither blocks nor log segments found. Creating new log.
18:29:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.570695  1841 tablet_bootstrap.cc:654] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Neither blocks nor log segments found. Creating new log.
18:29:49.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.574002  1842 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Log is configured to *not* fsync() on all Append() calls
18:29:49.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.574337  1841 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Log is configured to *not* fsync() on all Append() calls
18:29:49.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.574744  1843 tablet_bootstrap.cc:654] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Neither blocks nor log segments found. Creating new log.
18:29:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.579282  1843 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Log is configured to *not* fsync() on all Append() calls
18:29:49.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.582495  1842 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: No bootstrap required, opened a new log
18:29:49.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.583019  1842 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Time spent bootstrapping tablet: real 0.025s	user 0.004s	sys 0.015s
18:29:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.583845  1841 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: No bootstrap required, opened a new log
18:29:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.584426  1841 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Time spent bootstrapping tablet: real 0.035s	user 0.008s	sys 0.017s
18:29:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.587204  1843 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: No bootstrap required, opened a new log
18:29:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.588311  1843 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Time spent bootstrapping tablet: real 0.028s	user 0.003s	sys 0.020s
18:29:49.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.612390  1843 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:49.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.613083  1843 raft_consensus.cc:385] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:29:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.613430  1843 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9df350dc58374330a22867bfa403ed8d, State: Initialized, Role: FOLLOWER
18:29:49.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.614441  1843 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:49.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.617479  1841 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:49.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.618501  1841 raft_consensus.cc:385] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:29:49.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.618330  1842 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:49.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.618973  1841 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bacacb109f8247dcb77a540bd9e605a1, State: Initialized, Role: FOLLOWER
18:29:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.620389  1841 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:49.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.619171  1842 raft_consensus.cc:385] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:29:49.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.624301  1842 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8fdfc4241d34c848f5eb3275d07e1f6, State: Initialized, Role: FOLLOWER
18:29:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.625547  1842 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:49.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.636315  1841 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Time spent starting tablet: real 0.052s	user 0.039s	sys 0.009s
18:29:49.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.645856  1842 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Time spent starting tablet: real 0.063s	user 0.027s	sys 0.019s
18:29:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.671195  1847 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:29:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:49.672430  1532 tablet.cc:2406] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:29:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.671885  1847 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.672662  1843 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Time spent starting tablet: real 0.084s	user 0.036s	sys 0.038s
18:29:49.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.680474  1847 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:29:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.700431  1763 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1" is_pre_election: true
18:29:49.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.701578  1763 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9df350dc58374330a22867bfa403ed8d in term 0.
18:29:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.703702  1559 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, bacacb109f8247dcb77a540bd9e605a1; no voters: 
18:29:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.704716  1847 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 0 FOLLOWER]: Leader pre-election won for term 1
18:29:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.705071  1847 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:29:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.705375  1847 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 0 FOLLOWER]: Advancing to term 1
18:29:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.712785  1479 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" is_pre_election: true
18:29:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.713610  1479 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9df350dc58374330a22867bfa403ed8d in term 0.
18:29:49.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.717182  1847 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.718775  1847 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 1 election: Requested vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:29:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.719636  1479 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6"
18:29:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.719612  1763 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:29:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.720134  1763 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 0 FOLLOWER]: Advancing to term 1
18:29:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.720136  1479 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 0 FOLLOWER]: Advancing to term 1
18:29:49.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.728085  1763 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9df350dc58374330a22867bfa403ed8d in term 1.
18:29:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.729133  1559 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, bacacb109f8247dcb77a540bd9e605a1; no voters: 
18:29:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.730011  1479 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9df350dc58374330a22867bfa403ed8d in term 1.
18:29:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.730463  1847 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 1 FOLLOWER]: Leader election won for term 1
18:29:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.730903  1847 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 1 LEADER]: Becoming Leader. State: Replica: 9df350dc58374330a22867bfa403ed8d, State: Running, Role: LEADER
18:29:49.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.732043  1847 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.742949  1287 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d reported cstate change: term changed from 0 to 1, leader changed from <none> to 9df350dc58374330a22867bfa403ed8d (127.1.16.66). New cstate: current_term: 1 leader_uuid: "9df350dc58374330a22867bfa403ed8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: UNKNOWN } } }
18:29:49.792 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table ce2a5a51f13446bbab444c10f9739391
18:29:49.812 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:49.814235  1675 tablet.cc:2406] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:29:49.818 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299993, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:49.819 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=null) after lookup
18:29:49.829 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:29:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:49.829741  1815 tablet.cc:2406] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:29:49.832 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] Successfully connected to peer
18:29:49.920 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xda4c18a3, L:/127.0.0.1:43104 - R:/127.1.16.66:44881] using TOKEN/null
18:29:49.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.939775  1479 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 1 FOLLOWER]: Refusing update from remote peer 9df350dc58374330a22867bfa403ed8d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:29:49.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.940637  1763 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 1 FOLLOWER]: Refusing update from remote peer 9df350dc58374330a22867bfa403ed8d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:29:49.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.942137  1850 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:29:49.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.943033  1847 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
18:29:49.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.972057  1856 mvcc.cc:204] Tried to move back new op lower bound from 7305360957170835456 to 7305360956352061440. Current Snapshot: MvccSnapshot[applied={T|T < 7305360957170835456}]
18:29:49.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:49.988839  1857 mvcc.cc:204] Tried to move back new op lower bound from 7305360957170835456 to 7305360956352061440. Current Snapshot: MvccSnapshot[applied={T|T < 7305360957170835456}]
18:29:50.176 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:29:50.178 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.16.66:44881
18:29:50.180 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:29:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:50.180452  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 1531
18:29:50.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:29:50.213 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] handling channelInactive
18:29:50.213 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state READY due to: connection closed
18:29:50.214 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: connection closed
18:29:50.229 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:50.230 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:50.230 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [3ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1191657402(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@56178d39 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@230102dd -> wakeup thread Time-limited test)) after lookup
18:29:50.231 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:29:50.234 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.234 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.234 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.248 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:50.248 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:50.249 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [3ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [20ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [24ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [24ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [38ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1191657402(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@56178d39 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1111544029 after retry RPC@1605950536), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@230102dd -> wakeup thread Time-limited test -> (continuation of Deferred@1111544029 after retry RPC@1605950536))) after lookup
18:29:50.249 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:29:50.251 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.251 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.252 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.289 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:50.289 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:50.290 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [3ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [20ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [24ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [24ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [38ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [39ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [41ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [42ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [78ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1191657402(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@56178d39 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1111544029 after retry RPC@1605950536) -> (continuation of Deferred@1233034165 after retry RPC@1058263952), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@230102dd -> wakeup thread Time-limited test -> (continuation of Deferred@1111544029 after retry RPC@1605950536) -> (continuation of Deferred@1233034165 after retry RPC@1058263952))) after lookup
18:29:50.291 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:29:50.293 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.293 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.294 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.368 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:50.368 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:50.369 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=158), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [3ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [20ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [24ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [24ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [38ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [39ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [41ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [42ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [78ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [80ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [84ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [84ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [158ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1191657402(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@56178d39 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1111544029 after retry RPC@1605950536) -> (continuation of Deferred@1233034165 after retry RPC@1058263952) -> (continuation of Deferred@1880201417 after retry RPC@1265895526), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@230102dd -> wakeup thread Time-limited test -> (continuation of Deferred@1111544029 after retry RPC@1605950536) -> (continuation of Deferred@1233034165 after retry RPC@1058263952) -> (continuation of Deferred@1880201417 after retry RPC@1265895526))) after lookup
18:29:50.369 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:29:50.371 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.371 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.372 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.829 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:50.829 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:50.830 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=619), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [3ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [20ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [24ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [24ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [38ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [39ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [41ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [42ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [78ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [80ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [84ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [84ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [158ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [159ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [162ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [162ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [615ms] refreshing cache from master, [615ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [619ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1191657402(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@56178d39 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1111544029 after retry RPC@1605950536) -> (continuation of Deferred@1233034165 after retry RPC@1058263952) -> (continuation of Deferred@1880201417 after retry RPC@1265895526) -> (continuation of Deferred@364669152 after retry RPC@1281339945), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@230102dd -> wakeup thread Time-limited test -> (continuation of Deferred@1111544029 after retry RPC@1605950536) -> (continuation of Deferred@1233034165 after retry RPC@1058263952) -> (continuation of Deferred@1880201417 after retry RPC@1265895526) -> (continuation of Deferred@364669152 after retry RPC@1281339945))) after lookup
18:29:50.831 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:29:50.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:50.832 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:29:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.609658  1871 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 9df350dc58374330a22867bfa403ed8d)
18:29:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.610138  1871 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.612599  1871 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:29:51.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:51.616779  1699 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:29:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:51.621083  1699 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:29:51.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.625774  1479 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "bacacb109f8247dcb77a540bd9e605a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" is_pre_election: true
18:29:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.627259  1698 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bacacb109f8247dcb77a540bd9e605a1; no voters: 9df350dc58374330a22867bfa403ed8d, a8fdfc4241d34c848f5eb3275d07e1f6
18:29:51.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.627985  1871 raft_consensus.cc:2749] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
18:29:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.631130  1876 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 9df350dc58374330a22867bfa403ed8d)
18:29:51.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.631559  1876 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:51.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.634807  1876 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:29:51.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:51.640097  1413 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:29:51.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:51.644647  1413 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:29:51.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.645586  1763 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1" is_pre_election: true
18:29:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.646028  1763 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a8fdfc4241d34c848f5eb3275d07e1f6 in term 1.
18:29:51.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.646879  1413 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a8fdfc4241d34c848f5eb3275d07e1f6, bacacb109f8247dcb77a540bd9e605a1; no voters: 9df350dc58374330a22867bfa403ed8d
18:29:51.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.647578  1876 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 1 FOLLOWER]: Leader pre-election won for term 2
18:29:51.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.647882  1876 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 9df350dc58374330a22867bfa403ed8d)
18:29:51.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.648213  1876 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 1 FOLLOWER]: Advancing to term 2
18:29:51.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.652961  1876 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:51.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.654451  1876 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 2 election: Requested vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:29:51.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.655239  1763 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:29:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.655643  1763 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 1 FOLLOWER]: Advancing to term 2
18:29:51.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:51.658416  1413 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:29:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.661031  1763 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a8fdfc4241d34c848f5eb3275d07e1f6 in term 2.
18:29:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.661789  1413 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a8fdfc4241d34c848f5eb3275d07e1f6, bacacb109f8247dcb77a540bd9e605a1; no voters: 9df350dc58374330a22867bfa403ed8d
18:29:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.662359  1876 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 2 FOLLOWER]: Leader election won for term 2
18:29:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.664288  1876 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 2 LEADER]: Becoming Leader. State: Replica: a8fdfc4241d34c848f5eb3275d07e1f6, State: Running, Role: LEADER
18:29:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.665374  1876 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:51.675215  1288 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 reported cstate change: term changed from 1 to 2, leader changed from 9df350dc58374330a22867bfa403ed8d (127.1.16.66) to a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65). New cstate: current_term: 2 leader_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: HEALTHY } } }
18:29:51.989 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:51.990 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:51.991 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1780), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [3ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [20ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [24ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [24ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [38ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [39ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [41ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [42ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [78ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [80ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [84ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [84ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [154ms] refreshing cache from master, [154ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [158ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [159ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [162ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [162ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [615ms] refreshing cache from master, [615ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [619ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [620ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [622ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [622ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1774ms] refreshing cache from master, [1775ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1779ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1191657402(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@56178d39 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1111544029 after retry RPC@1605950536) -> (continuation of Deferred@1233034165 after retry RPC@1058263952) -> (continuation of Deferred@1880201417 after retry RPC@1265895526) -> (continuation of Deferred@364669152 after retry RPC@1281339945) -> (continuation of Deferred@422415963 after retry RPC@545275530), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@230102dd -> wakeup thread Time-limited test -> (continuation of Deferred@1111544029 after retry RPC@1605950536) -> (continuation of Deferred@1233034165 after retry RPC@1058263952) -> (continuation of Deferred@1880201417 after retry RPC@1265895526) -> (continuation of Deferred@364669152 after retry RPC@1281339945) -> (continuation of Deferred@422415963 after retry RPC@545275530))) after lookup
18:29:51.991 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:29:51.993 [DEBUG - kudu-nio-1] (Connection.java:757) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] Successfully connected to peer
18:29:52.020 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa16bd566, L:/127.0.0.1:49894 - R:/127.1.16.65:45335] using TOKEN/null
18:29:52.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:52.031905  1763 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 2 FOLLOWER]: Refusing update from remote peer a8fdfc4241d34c848f5eb3275d07e1f6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
18:29:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:52.033892  1876 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
18:29:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:52.037602  1413 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 -> Peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Couldn't send request to peer 9df350dc58374330a22867bfa403ed8d. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:29:52.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:52.052665  1894 mvcc.cc:204] Tried to move back new op lower bound from 7305360965745164288 to 7305360964268662784. Current Snapshot: MvccSnapshot[applied={T|T < 7305360965745164288}]
18:29:52.093 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.16.66:44881
18:29:52.094 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:52.095901  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/logs
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:29:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.66:44881
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.66
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36159
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.66
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:29:52.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:52.411598  1900 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:29:52.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:52.412063  1900 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:29:52.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:52.412616  1900 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:29:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:52.445608  1900 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:29:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:52.446625  1900 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.66
18:29:52.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:52.487879  1900 tablet_server_runner.cc:78] Tablet server non-default flags:
18:29:52.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:52.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:52.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.66:44881
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.66
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36159
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.66
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/logs
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:29:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:29:52.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:29:52.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:29:52.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:29:52.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:29:52.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:52.490015  1900 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:29:52.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:52.492295  1900 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:29:52.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:52.505937  1908 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:54.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:53.907771  1907 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 1900
18:29:54.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:52.507270  1909 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:54.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:54.332335  1900 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.826s	user 0.617s	sys 1.201s
18:29:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:54.335146  1900 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.829s	user 0.618s	sys 1.201s
18:29:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:54.335425  1910 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1829 milliseconds
18:29:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:54.336748  1911 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:29:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.337174  1900 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:29:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.340287  1900 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:29:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.342816  1900 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:29:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.344463  1900 hybrid_clock.cc:648] HybridClock initialized: now 1783535394344418 us; error 35 us; skew 500 ppm
18:29:54.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.345643  1900 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:29:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.352005  1900 webserver.cc:534] Webserver started at http://127.1.16.66:36159/ using document root <none> and password file <none>
18:29:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.352960  1900 fs_manager.cc:362] Metadata directory not provided
18:29:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.353227  1900 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:29:54.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.361639  1900 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.001s
18:29:54.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.366220  1921 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:29:54.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.367321  1900 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
18:29:54.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.367710  1900 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:29:54.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9df350dc58374330a22867bfa403ed8d"
18:29:54.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:45 on dist-test-slave-l44q"
18:29:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.369822  1900 fs_report.cc:389] FS layout report
18:29:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:29:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:29:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:29:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/data
18:29:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:29:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:29:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:29:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:29:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:29:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:29:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:29:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:29:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:29:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:29:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:29:54.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.430310  1900 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:29:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.431886  1900 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:29:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.432480  1900 kserver.cc:163] Server-wide thread pool size limit: 3276
18:29:54.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.435117  1900 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:29:54.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.441195  1928 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:29:54.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.449470  1900 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:29:54.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.449683  1900 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
18:29:54.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.449908  1900 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:29:54.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.454753  1900 ts_tablet_manager.cc:616] Registered 1 tablets
18:29:54.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.454998  1900 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
18:29:54.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.455331  1928 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap starting.
18:29:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:54.527521  1413 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 -> Peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Couldn't send request to peer 9df350dc58374330a22867bfa403ed8d. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
18:29:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.530645  1928 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Log is configured to *not* fsync() on all Append() calls
18:29:54.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.574965  1928 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:29:54.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.576246  1928 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap complete.
18:29:54.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.578599  1928 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Time spent bootstrapping tablet: real 0.124s	user 0.075s	sys 0.043s
18:29:54.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.608558  1928 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:54.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.612658  1928 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9df350dc58374330a22867bfa403ed8d, State: Initialized, Role: FOLLOWER
18:29:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.614063  1928 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:54.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.619355  1928 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Time spent starting tablet: real 0.040s	user 0.041s	sys 0.001s
18:29:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.635823  1900 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.66:44881
18:29:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.636077  2038 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.66:44881 every 8 connection(s)
18:29:54.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.638417  1900 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:29:54.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.642063  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 1900
18:29:54.643 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:29:54.667 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:29:54.667 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.16.65:45335
18:29:54.668 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:29:54.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.668746  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 1369
18:29:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.678028  2039 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:29:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.678535  2039 heartbeater.cc:461] Registering TS with master...
18:29:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.679889  2039 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:29:54.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.684998  1288 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:29:54.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.688616  1288 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.66:47811
18:29:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.697914  2041 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:29:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.698243  2041 heartbeater.cc:461] Registering TS with master...
18:29:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.698647  2040 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:29:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.698968  2040 heartbeater.cc:461] Registering TS with master...
18:29:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.698938  2041 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:29:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.699613  2040 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:29:54.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.703109  1208 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:29:54.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:54.703120  1133 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:29:54.720 [DEBUG - kudu-nio-1] (Connection.java:239) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] handling channelInactive
18:29:54.721 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state READY due to: connection closed
18:29:54.722 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:29:54.723 [DEBUG - Time-limited test] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:29:54.724 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.724 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.724 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.748 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:54.749 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:54.750 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [3ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@489010879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@3cf0bfff -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4df9ee82 -> wakeup thread Time-limited test)) after lookup
18:29:54.751 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:29:54.752 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.752 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.752 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.767 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:54.768 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:54.768 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [3ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [30ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [30ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@489010879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@3cf0bfff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4df9ee82 -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009))) after lookup
18:29:54.769 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:29:54.770 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.770 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.770 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.788 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:54.788 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:54.789 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=66), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [3ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [30ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [30ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [46ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [48ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [48ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [66ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@489010879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@3cf0bfff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009) -> (continuation of Deferred@853170675 after retry RPC@476751719), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4df9ee82 -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009) -> (continuation of Deferred@853170675 after retry RPC@476751719))) after lookup
18:29:54.789 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:29:54.790 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.790 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.790 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.807 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:54.808 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:54.808 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=85), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [3ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [30ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [30ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [46ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [48ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [48ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [66ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [67ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [68ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [69ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [85ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@489010879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@3cf0bfff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009) -> (continuation of Deferred@853170675 after retry RPC@476751719) -> (continuation of Deferred@1146508828 after retry RPC@231637102), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4df9ee82 -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009) -> (continuation of Deferred@853170675 after retry RPC@476751719) -> (continuation of Deferred@1146508828 after retry RPC@231637102))) after lookup
18:29:54.809 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:29:54.810 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.810 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:54.810 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:55.308 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:55.309 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:55.309 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=586), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [3ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [30ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [30ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [46ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [48ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [48ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [66ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [67ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [68ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [69ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [85ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [86ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [88ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [88ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [582ms] refreshing cache from master, [582ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [586ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@489010879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@3cf0bfff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009) -> (continuation of Deferred@853170675 after retry RPC@476751719) -> (continuation of Deferred@1146508828 after retry RPC@231637102) -> (continuation of Deferred@1605578555 after retry RPC@637995347), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4df9ee82 -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009) -> (continuation of Deferred@853170675 after retry RPC@476751719) -> (continuation of Deferred@1146508828 after retry RPC@231637102) -> (continuation of Deferred@1605578555 after retry RPC@637995347))) after lookup
18:29:55.310 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:29:55.311 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:55.312 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:55.312 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:55.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:55.693186  2039 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:29:56.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.060777  2052 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader a8fdfc4241d34c848f5eb3275d07e1f6)
18:29:56.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.061422  2052 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:56.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.063252  2052 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:29:56.071 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:56.071 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:29:56.072 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1349), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [3ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [30ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [30ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [46ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [48ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [48ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [66ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [67ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [68ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [69ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [85ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [86ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [88ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [88ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [582ms] refreshing cache from master, [582ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [586ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [588ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [590ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [590ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [1342ms] refreshing cache from master, [1343ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1349ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@489010879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@3cf0bfff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009) -> (continuation of Deferred@853170675 after retry RPC@476751719) -> (continuation of Deferred@1146508828 after retry RPC@231637102) -> (continuation of Deferred@1605578555 after retry RPC@637995347) -> (continuation of Deferred@1194632520 after retry RPC@662989192), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4df9ee82 -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009) -> (continuation of Deferred@853170675 after retry RPC@476751719) -> (continuation of Deferred@1146508828 after retry RPC@231637102) -> (continuation of Deferred@1605578555 after retry RPC@637995347) -> (continuation of Deferred@1194632520 after retry RPC@662989192))) after lookup
18:29:56.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:56.073406  1698 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:29:56.074 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:29:56.075 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:56.075 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:56.075 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:29:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.078513  1993 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "bacacb109f8247dcb77a540bd9e605a1" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "9df350dc58374330a22867bfa403ed8d" is_pre_election: true
18:29:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.079268  1993 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bacacb109f8247dcb77a540bd9e605a1 in term 1.
18:29:56.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.080572  1699 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, bacacb109f8247dcb77a540bd9e605a1; no voters: a8fdfc4241d34c848f5eb3275d07e1f6
18:29:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.081241  2052 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 2 FOLLOWER]: Leader pre-election won for term 3
18:29:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.081555  2052 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 2 FOLLOWER]: Starting leader election (detected failure of leader a8fdfc4241d34c848f5eb3275d07e1f6)
18:29:56.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.081779  2052 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 2 FOLLOWER]: Advancing to term 3
18:29:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.086377  2052 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.088138  2052 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 3 election: Requested vote from peers 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:29:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.088944  1993 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "bacacb109f8247dcb77a540bd9e605a1" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "9df350dc58374330a22867bfa403ed8d"
18:29:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.089546  1993 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 1 FOLLOWER]: Advancing to term 3
18:29:56.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:56.091584  1698 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:29:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.098129  1993 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bacacb109f8247dcb77a540bd9e605a1 in term 3.
18:29:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.099016  1699 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, bacacb109f8247dcb77a540bd9e605a1; no voters: a8fdfc4241d34c848f5eb3275d07e1f6
18:29:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.099741  2052 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 3 FOLLOWER]: Leader election won for term 3
18:29:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.101097  2052 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 3 LEADER]: Becoming Leader. State: Replica: bacacb109f8247dcb77a540bd9e605a1, State: Running, Role: LEADER
18:29:56.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.102252  2052 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:29:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.110984  1288 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 reported cstate change: term changed from 2 to 3, leader changed from a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65) to bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67). New cstate: current_term: 3 leader_uuid: "bacacb109f8247dcb77a540bd9e605a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: UNKNOWN } } }
18:29:56.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.525146  1993 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 3 FOLLOWER]: Refusing update from remote peer bacacb109f8247dcb77a540bd9e605a1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
18:29:56.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:56.526480  2052 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
18:29:56.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:56.562244  1698 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 -> Peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Couldn't send request to peer a8fdfc4241d34c848f5eb3275d07e1f6. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:29:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:57.024017  1698 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111) [suppressed 3 similar messages]
18:29:58.448 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:29:58.449 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:29:58.449 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=3726), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [3ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [30ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [30ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [46ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [48ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [48ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [66ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [67ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [68ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [69ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [85ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [86ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [88ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [88ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [582ms] refreshing cache from master, [582ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [586ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [588ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [590ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [590ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [1342ms] refreshing cache from master, [1343ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1349ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [1350ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [1353ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [1353ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [3722ms] refreshing cache from master, [3722ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [3726ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@489010879(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@3cf0bfff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009) -> (continuation of Deferred@853170675 after retry RPC@476751719) -> (continuation of Deferred@1146508828 after retry RPC@231637102) -> (continuation of Deferred@1605578555 after retry RPC@637995347) -> (continuation of Deferred@1194632520 after retry RPC@662989192) -> (continuation of Deferred@1761850094 after retry RPC@815733242), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4df9ee82 -> wakeup thread Time-limited test -> (continuation of Deferred@1365301113 after retry RPC@1399828009) -> (continuation of Deferred@853170675 after retry RPC@476751719) -> (continuation of Deferred@1146508828 after retry RPC@231637102) -> (continuation of Deferred@1605578555 after retry RPC@637995347) -> (continuation of Deferred@1194632520 after retry RPC@662989192) -> (continuation of Deferred@1761850094 after retry RPC@815733242))) after lookup
18:29:58.450 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:29:58.451 [DEBUG - kudu-nio-1] (Connection.java:757) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] Successfully connected to peer
18:29:58.470 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xfb685e0d, L:/127.0.0.1:43030 - R:/127.1.16.67:37285] using TOKEN/null
18:29:58.514 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.16.65:45335
18:29:58.516 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:29:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:58.517352  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:29:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:29:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data
18:29:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:29:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:29:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/logs
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.65:45335
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.65
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44059
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.65
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:29:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:58.840077  2095 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:29:58.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:58.840767  2095 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:29:58.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:58.841466  2095 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:29:58.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:58.877343  2095 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:29:58.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:58.878360  2095 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.65
18:29:58.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:58.903172  1698 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 -> Peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Couldn't send request to peer a8fdfc4241d34c848f5eb3275d07e1f6. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:58.915760  2095 tablet_server_runner.cc:78] Tablet server non-default flags:
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.65:45335
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.65
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44059
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:29:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.65
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/logs
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:29:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:58.917290  2095 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:29:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:29:58.919020  2095 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:29:58.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:58.931159  2101 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:00.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:29:58.932534  2102 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:00.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:00.238063  2104 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:00.240902  2103 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1303 milliseconds
18:30:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.241032  2095 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:30:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.242451  2095 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:30:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.245193  2095 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:30:00.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.246630  2095 hybrid_clock.cc:648] HybridClock initialized: now 1783535400246580 us; error 53 us; skew 500 ppm
18:30:00.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.247488  2095 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:30:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.254097  2095 webserver.cc:534] Webserver started at http://127.1.16.65:44059/ using document root <none> and password file <none>
18:30:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.254931  2095 fs_manager.cc:362] Metadata directory not provided
18:30:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.255121  2095 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:30:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.262625  2095 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
18:30:00.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.267719  2114 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:30:00.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.268937  2095 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
18:30:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.269212  2095 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8fdfc4241d34c848f5eb3275d07e1f6"
18:30:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:42 on dist-test-slave-l44q"
18:30:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.271260  2095 fs_report.cc:389] FS layout report
18:30:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:30:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/data
18:30:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:30:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:30:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:30:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:30:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:30:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:30:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:30:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:30:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:30:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:30:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:30:00.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.327943  2095 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:30:00.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.329412  2095 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:30:00.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.329825  2095 kserver.cc:163] Server-wide thread pool size limit: 3276
18:30:00.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.332418  2095 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:30:00.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.337848  2121 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:30:00.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.345978  2095 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:30:00.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.346179  2095 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.000s
18:30:00.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.346392  2095 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:30:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.351064  2095 ts_tablet_manager.cc:616] Registered 1 tablets
18:30:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.351234  2095 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
18:30:00.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.352185  2121 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Bootstrap starting.
18:30:00.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.417765  2121 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Log is configured to *not* fsync() on all Append() calls
18:30:00.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.453272  2121 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:30:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.454213  2121 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Bootstrap complete.
18:30:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.456135  2121 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Time spent bootstrapping tablet: real 0.105s	user 0.072s	sys 0.028s
18:30:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.474169  2121 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.477489  2121 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8fdfc4241d34c848f5eb3275d07e1f6, State: Initialized, Role: FOLLOWER
18:30:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.478611  2121 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:00.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.482355  2121 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Time spent starting tablet: real 0.026s	user 0.021s	sys 0.004s
18:30:00.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.523768  2095 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.65:45335
18:30:00.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.524083  2229 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.65:45335 every 8 connection(s)
18:30:00.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.526815  2095 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:30:00.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.534736  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 2095
18:30:00.535 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:00.561 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:30:00.562 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.16.67:37285
18:30:00.562 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:30:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.563265  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 1677
18:30:00.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.576123  2230 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:30:00.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.576665  2230 heartbeater.cc:461] Registering TS with master...
18:30:00.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.577757  2230 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:30:00.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.581738  2231 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:30:00.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.581912  1288 ts_manager.cc:194] Re-registered known tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:00.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.582074  2231 heartbeater.cc:461] Registering TS with master...
18:30:00.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.583000  2231 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:30:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.585001  1288 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.65:42771
18:30:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.585868  2232 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:30:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.586217  2232 heartbeater.cc:461] Registering TS with master...
18:30:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.587131  2232 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:30:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.587566  1208 ts_manager.cc:194] Re-registered known tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:00.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:00.591941  1133 ts_manager.cc:194] Re-registered known tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:00.602 [DEBUG - kudu-nio-1] (Connection.java:239) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] handling channelInactive
18:30:00.602 [DEBUG - kudu-nio-1] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state READY due to: connection closed
18:30:00.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:00.603 [DEBUG - Time-limited test] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:00.605 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.605 [DEBUG - kudu-nio-2] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.605 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.629 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:00.629 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:00.629 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [2ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [2ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@655907766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6f80d666 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ed1bda -> wakeup thread Time-limited test)) after lookup
18:30:00.630 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:00.631 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.631 [DEBUG - kudu-nio-3] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.631 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.648 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:00.648 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:00.649 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [2ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [2ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [26ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [28ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [28ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@655907766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6f80d666 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@490964020 after retry RPC@2006609805), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ed1bda -> wakeup thread Time-limited test -> (continuation of Deferred@490964020 after retry RPC@2006609805))) after lookup
18:30:00.649 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:00.650 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.650 [DEBUG - kudu-nio-4] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.650 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.689 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:00.689 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:00.690 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=86), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [2ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [2ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [26ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [28ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [28ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [46ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [47ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [47ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [85ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@655907766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6f80d666 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@490964020 after retry RPC@2006609805) -> (continuation of Deferred@308154275 after retry RPC@907869282), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ed1bda -> wakeup thread Time-limited test -> (continuation of Deferred@490964020 after retry RPC@2006609805) -> (continuation of Deferred@308154275 after retry RPC@907869282))) after lookup
18:30:00.690 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:00.691 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.691 [DEBUG - kudu-nio-5] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.692 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.809 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:00.809 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:00.810 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=207), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [2ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [2ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [26ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [28ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [28ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [46ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [47ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [47ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [85ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [87ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [89ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [89ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [201ms] refreshing cache from master, [201ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [205ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@655907766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6f80d666 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@490964020 after retry RPC@2006609805) -> (continuation of Deferred@308154275 after retry RPC@907869282) -> (continuation of Deferred@509242936 after retry RPC@1333030525), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ed1bda -> wakeup thread Time-limited test -> (continuation of Deferred@490964020 after retry RPC@2006609805) -> (continuation of Deferred@308154275 after retry RPC@907869282) -> (continuation of Deferred@509242936 after retry RPC@1333030525))) after lookup
18:30:00.811 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:00.812 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.812 [DEBUG - kudu-nio-6] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:00.812 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:01.089 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:01.089 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:01.089 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=486), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [2ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [2ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [26ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [28ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [28ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [46ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [47ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [47ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [85ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [87ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [89ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [89ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [201ms] refreshing cache from master, [201ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [205ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [207ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [209ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [209ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [481ms] refreshing cache from master, [481ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [485ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@655907766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6f80d666 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@490964020 after retry RPC@2006609805) -> (continuation of Deferred@308154275 after retry RPC@907869282) -> (continuation of Deferred@509242936 after retry RPC@1333030525) -> (continuation of Deferred@795167800 after retry RPC@1133430831), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ed1bda -> wakeup thread Time-limited test -> (continuation of Deferred@490964020 after retry RPC@2006609805) -> (continuation of Deferred@308154275 after retry RPC@907869282) -> (continuation of Deferred@509242936 after retry RPC@1333030525) -> (continuation of Deferred@795167800 after retry RPC@1133430831))) after lookup
18:30:01.090 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:01.091 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:01.091 [DEBUG - kudu-nio-7] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:01.091 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:01.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.594128  2230 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:30:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.734354  2241 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 3 FOLLOWER]: Starting pre-election (detected failure of leader bacacb109f8247dcb77a540bd9e605a1)
18:30:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.734831  2241 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:01.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.737244  2241 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:01.740115  1925 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111)
18:30:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:01.743582  1925 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111)
18:30:01.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.749356  2184 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" is_pre_election: true
18:30:01.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.749980  2184 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9df350dc58374330a22867bfa403ed8d in term 2.
18:30:01.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.751245  1924 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, a8fdfc4241d34c848f5eb3275d07e1f6; no voters: bacacb109f8247dcb77a540bd9e605a1
18:30:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.751964  2241 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 3 FOLLOWER]: Leader pre-election won for term 4
18:30:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.752316  2241 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 3 FOLLOWER]: Starting leader election (detected failure of leader bacacb109f8247dcb77a540bd9e605a1)
18:30:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.752662  2241 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 3 FOLLOWER]: Advancing to term 4
18:30:01.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.757000  2241 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:01.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.758463  2241 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 4 election: Requested vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:01.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.759404  2184 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6"
18:30:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.759845  2184 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 2 FOLLOWER]: Advancing to term 4
18:30:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:01.761694  1925 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111)
18:30:01.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.766700  2184 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9df350dc58374330a22867bfa403ed8d in term 4.
18:30:01.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.767642  1924 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, a8fdfc4241d34c848f5eb3275d07e1f6; no voters: bacacb109f8247dcb77a540bd9e605a1
18:30:01.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.768254  2241 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 4 FOLLOWER]: Leader election won for term 4
18:30:01.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.769905  2241 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 4 LEADER]: Becoming Leader. State: Replica: 9df350dc58374330a22867bfa403ed8d, State: Running, Role: LEADER
18:30:01.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.770661  2241 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:01.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:01.779587  1288 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d reported cstate change: term changed from 3 to 4, leader changed from bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67) to 9df350dc58374330a22867bfa403ed8d (127.1.16.66). New cstate: current_term: 4 leader_uuid: "9df350dc58374330a22867bfa403ed8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: UNKNOWN } } }
18:30:02.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:02.190713  1925 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d -> Peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Couldn't send request to peer bacacb109f8247dcb77a540bd9e605a1. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:30:02.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:02.224507  2184 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 4 FOLLOWER]: Refusing update from remote peer 9df350dc58374330a22867bfa403ed8d: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
18:30:02.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:02.226049  2241 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
18:30:02.908 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:02.909 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:02.909 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=2306), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [2ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [2ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [26ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [28ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [28ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [46ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [47ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [47ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [85ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [87ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [89ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [89ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [201ms] refreshing cache from master, [201ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [205ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [207ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [209ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [209ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [481ms] refreshing cache from master, [481ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [485ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [487ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [488ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [488ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [2301ms] refreshing cache from master, [2301ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [2305ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@655907766(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6f80d666 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@490964020 after retry RPC@2006609805) -> (continuation of Deferred@308154275 after retry RPC@907869282) -> (continuation of Deferred@509242936 after retry RPC@1333030525) -> (continuation of Deferred@795167800 after retry RPC@1133430831) -> (continuation of Deferred@1210184022 after retry RPC@2103150541), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2ed1bda -> wakeup thread Time-limited test -> (continuation of Deferred@490964020 after retry RPC@2006609805) -> (continuation of Deferred@308154275 after retry RPC@907869282) -> (continuation of Deferred@509242936 after retry RPC@1333030525) -> (continuation of Deferred@795167800 after retry RPC@1133430831) -> (continuation of Deferred@1210184022 after retry RPC@2103150541))) after lookup
18:30:02.910 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:02.911 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] Successfully connected to peer
18:30:02.934 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa9589445, L:/127.0.0.1:53140 - R:/127.1.16.66:44881] using TOKEN/null
18:30:02.989 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.16.67:37285
18:30:02.989 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:02.990865  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/logs
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:30:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.67:37285
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.67
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37737
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.67
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:30:03.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:03.313784  2276 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:30:03.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:03.314344  2276 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:30:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:03.314986  2276 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:30:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:03.347545  2276 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:30:03.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:03.348641  2276 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.67
18:30:03.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:03.386178  2276 tablet_server_runner.cc:78] Tablet server non-default flags:
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.67:37285
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.67
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37737
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.67
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/logs
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:30:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:03.387609  2276 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:30:03.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:03.389206  2276 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:30:03.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:03.403208  2282 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:04.539036  1240 debug-util.cc:398] Leaking SignalData structure 0x7b0800070060 after lost signal to thread 1175
18:30:04.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:04.539994  1240 debug-util.cc:398] Leaking SignalData structure 0x7b0800095120 after lost signal to thread 1243
18:30:04.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:04.613854  1925 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d -> Peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Couldn't send request to peer bacacb109f8247dcb77a540bd9e605a1. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
18:30:04.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:03.404562  2283 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:04.790210  2285 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:04.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:04.793183  2284 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1388 milliseconds
18:30:04.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.793296  2276 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:30:04.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.794559  2276 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:30:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.799141  2276 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:30:04.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.800575  2276 hybrid_clock.cc:648] HybridClock initialized: now 1783535404800487 us; error 94 us; skew 500 ppm
18:30:04.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.801453  2276 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:30:04.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.808691  2276 webserver.cc:534] Webserver started at http://127.1.16.67:37737/ using document root <none> and password file <none>
18:30:04.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.809630  2276 fs_manager.cc:362] Metadata directory not provided
18:30:04.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.809890  2276 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:30:04.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.818940  2276 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.002s	sys 0.003s
18:30:04.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.824611  2296 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:30:04.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.825783  2276 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
18:30:04.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.826059  2276 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:47 on dist-test-slave-l44q"
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.827915  2276 fs_report.cc:389] FS layout report
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/data
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:30:04.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:30:04.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:30:04.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:30:04.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.891979  2276 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:30:04.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.893756  2276 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:30:04.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.894248  2276 kserver.cc:163] Server-wide thread pool size limit: 3276
18:30:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.897593  2276 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:30:04.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.904567  2303 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:30:04.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.912874  2276 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:30:04.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.913112  2276 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.001s
18:30:04.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.913407  2276 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:30:04.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.917999  2276 ts_tablet_manager.cc:616] Registered 1 tablets
18:30:04.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.918251  2276 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
18:30:04.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.918612  2303 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Bootstrap starting.
18:30:04.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:04.995317  2303 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Log is configured to *not* fsync() on all Append() calls
18:30:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.072959  2303 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:30:05.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.074878  2303 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Bootstrap complete.
18:30:05.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.077358  2303 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Time spent bootstrapping tablet: real 0.159s	user 0.115s	sys 0.034s
18:30:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.115844  2303 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.119781  2303 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: bacacb109f8247dcb77a540bd9e605a1, State: Initialized, Role: FOLLOWER
18:30:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.121165  2303 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:05.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.133585  2303 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Time spent starting tablet: real 0.056s	user 0.023s	sys 0.022s
18:30:05.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.231233  2276 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.67:37285
18:30:05.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.231459  2411 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.67:37285 every 8 connection(s)
18:30:05.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.234174  2276 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:30:05.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.244848  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 2276
18:30:05.245 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:05.293 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:30:05.299 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.16.66:44881
18:30:05.300 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:30:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.300611  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 1900
18:30:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.306293  2412 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:30:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.306779  2412 heartbeater.cc:461] Registering TS with master...
18:30:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.308132  2412 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:30:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.314922  1288 ts_manager.cc:194] Re-registered known tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:30:05.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.317427  1288 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.67:34251
18:30:05.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.324527  2413 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:30:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.324865  2413 heartbeater.cc:461] Registering TS with master...
18:30:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.325462  2413 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:30:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.326929  2416 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:30:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.327239  2416 heartbeater.cc:461] Registering TS with master...
18:30:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.327839  2416 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:30:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.328866  1209 ts_manager.cc:194] Re-registered known tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:30:05.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:05.331370  1134 ts_manager.cc:194] Re-registered known tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:30:05.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:05.357194  1118 connection.cc:573] server connection from 127.1.16.66:53357 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
18:30:05.358 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] handling channelInactive
18:30:05.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state READY due to: connection closed
18:30:05.359 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:05.359 [DEBUG - Time-limited test] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:05.360 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.361 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.361 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.368 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:05.368 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:05.369 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1523870069(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c70de9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41336491 -> wakeup thread Time-limited test)) after lookup
18:30:05.369 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:05.370 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.371 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.371 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.388 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:05.389 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:05.389 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [10ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [12ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [12ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1523870069(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c70de9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1659529901 after retry RPC@942120333), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41336491 -> wakeup thread Time-limited test -> (continuation of Deferred@1659529901 after retry RPC@942120333))) after lookup
18:30:05.389 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:05.390 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.390 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.391 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.407 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:05.408 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:05.408 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [10ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [12ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [12ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [30ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [31ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [32ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [48ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1523870069(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c70de9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1659529901 after retry RPC@942120333) -> (continuation of Deferred@1113056468 after retry RPC@1176031974), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41336491 -> wakeup thread Time-limited test -> (continuation of Deferred@1659529901 after retry RPC@942120333) -> (continuation of Deferred@1113056468 after retry RPC@1176031974))) after lookup
18:30:05.409 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:05.410 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.410 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.410 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.448 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:05.448 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:05.448 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=89), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [10ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [12ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [12ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [30ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [31ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [32ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [48ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [49ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [51ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [51ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [88ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1523870069(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c70de9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1659529901 after retry RPC@942120333) -> (continuation of Deferred@1113056468 after retry RPC@1176031974) -> (continuation of Deferred@1201909150 after retry RPC@1194117230), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41336491 -> wakeup thread Time-limited test -> (continuation of Deferred@1659529901 after retry RPC@942120333) -> (continuation of Deferred@1113056468 after retry RPC@1176031974) -> (continuation of Deferred@1201909150 after retry RPC@1194117230))) after lookup
18:30:05.449 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:05.450 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.451 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.451 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.828 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:05.828 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:05.828 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=469), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [10ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [12ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [12ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [30ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [31ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [32ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [48ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [49ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [51ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [51ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [88ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [90ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [92ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [92ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [465ms] refreshing cache from master, [465ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [468ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1523870069(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c70de9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1659529901 after retry RPC@942120333) -> (continuation of Deferred@1113056468 after retry RPC@1176031974) -> (continuation of Deferred@1201909150 after retry RPC@1194117230) -> (continuation of Deferred@1705532877 after retry RPC@1155958446), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41336491 -> wakeup thread Time-limited test -> (continuation of Deferred@1659529901 after retry RPC@942120333) -> (continuation of Deferred@1113056468 after retry RPC@1176031974) -> (continuation of Deferred@1201909150 after retry RPC@1194117230) -> (continuation of Deferred@1705532877 after retry RPC@1155958446))) after lookup
18:30:05.829 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:05.830 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.830 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:05.830 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.322168  2412 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:30:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.448589  2424 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 9df350dc58374330a22867bfa403ed8d)
18:30:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.448998  2424 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.451125  2424 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:06.458598  2118 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:30:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.464021  2367 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1" is_pre_election: true
18:30:06.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.464962  2367 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a8fdfc4241d34c848f5eb3275d07e1f6 in term 3.
18:30:06.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.467250  2118 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a8fdfc4241d34c848f5eb3275d07e1f6, bacacb109f8247dcb77a540bd9e605a1; no voters: 
18:30:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.468016  2424 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 4 FOLLOWER]: Leader pre-election won for term 5
18:30:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:06.468086  2118 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:30:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.468395  2424 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 9df350dc58374330a22867bfa403ed8d)
18:30:06.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.468740  2424 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 4 FOLLOWER]: Advancing to term 5
18:30:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.473150  2424 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.474697  2424 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 5 election: Requested vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:06.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.475481  2367 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:30:06.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.476207  2367 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 3 FOLLOWER]: Advancing to term 5
18:30:06.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:06.478365  2118 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:30:06.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.487484  2367 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a8fdfc4241d34c848f5eb3275d07e1f6 in term 5.
18:30:06.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.488737  2118 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a8fdfc4241d34c848f5eb3275d07e1f6, bacacb109f8247dcb77a540bd9e605a1; no voters: 9df350dc58374330a22867bfa403ed8d
18:30:06.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.489354  2424 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 5 FOLLOWER]: Leader election won for term 5
18:30:06.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.491174  2424 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 5 LEADER]: Becoming Leader. State: Replica: a8fdfc4241d34c848f5eb3275d07e1f6, State: Running, Role: LEADER
18:30:06.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.491943  2424 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:06.501510  1288 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 reported cstate change: term changed from 4 to 5, leader changed from 9df350dc58374330a22867bfa403ed8d (127.1.16.66) to a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65). New cstate: current_term: 5 leader_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: HEALTHY } } }
18:30:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:06.876453  2118 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 -> Peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Couldn't send request to peer 9df350dc58374330a22867bfa403ed8d. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:30:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:07.016494  2367 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 5 FOLLOWER]: Refusing update from remote peer a8fdfc4241d34c848f5eb3275d07e1f6: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
18:30:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:07.017786  2424 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
18:30:07.028 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:07.029 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:07.029 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1670), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [10ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [12ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [12ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [30ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [31ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [32ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [48ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [49ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [51ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [51ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [88ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [90ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [92ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [92ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [465ms] refreshing cache from master, [465ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [468ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [470ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [471ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [472ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1665ms] refreshing cache from master, [1665ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1669ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1523870069(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c70de9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1659529901 after retry RPC@942120333) -> (continuation of Deferred@1113056468 after retry RPC@1176031974) -> (continuation of Deferred@1201909150 after retry RPC@1194117230) -> (continuation of Deferred@1705532877 after retry RPC@1155958446) -> (continuation of Deferred@1209077775 after retry RPC@1194920586), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41336491 -> wakeup thread Time-limited test -> (continuation of Deferred@1659529901 after retry RPC@942120333) -> (continuation of Deferred@1113056468 after retry RPC@1176031974) -> (continuation of Deferred@1201909150 after retry RPC@1194117230) -> (continuation of Deferred@1705532877 after retry RPC@1155958446) -> (continuation of Deferred@1209077775 after retry RPC@1194920586))) after lookup
18:30:07.030 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:07.039 [DEBUG - kudu-nio-7] (Connection.java:757) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] Successfully connected to peer
18:30:07.084 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x242f08e0, L:/127.0.0.1:48560 - R:/127.1.16.65:45335] using TOKEN/null
18:30:07.151 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.16.66:44881
18:30:07.151 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:07.152961  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/logs
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.66:44881
18:30:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.66
18:30:07.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:07.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36159
18:30:07.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.66
18:30:07.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:07.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:07.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:07.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:30:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:07.464897  2453 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:30:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:07.465374  2453 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:30:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:07.465834  2453 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:30:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:07.498526  2453 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:30:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:07.499946  2453 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.66
18:30:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:07.540267  2453 tablet_server_runner.cc:78] Tablet server non-default flags:
18:30:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.66:44881
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.66
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36159
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.66
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/logs
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:30:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:30:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:07.541648  2453 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:30:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:07.543322  2453 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:30:07.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:07.556584  2462 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:07.556936  2463 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:08.862370  2453 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.307s	user 0.003s	sys 0.004s
18:30:08.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:08.862740  2453 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.308s	user 0.003s	sys 0.004s
18:30:08.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.865454  2453 server_base.cc:1061] running on GCE node
18:30:08.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:08.866699  2472 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:08.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.867966  2453 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:30:08.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.870546  2453 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:30:08.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.871963  2453 hybrid_clock.cc:648] HybridClock initialized: now 1783535408871910 us; error 50 us; skew 500 ppm
18:30:08.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.872864  2453 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:30:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.881189  2453 webserver.cc:534] Webserver started at http://127.1.16.66:36159/ using document root <none> and password file <none>
18:30:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.882099  2453 fs_manager.cc:362] Metadata directory not provided
18:30:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.882341  2453 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:30:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.890904  2453 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.005s	sys 0.000s
18:30:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.895865  2477 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:30:08.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.897225  2453 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
18:30:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.897634  2453 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9df350dc58374330a22867bfa403ed8d"
18:30:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:45 on dist-test-slave-l44q"
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.899606  2453 fs_report.cc:389] FS layout report
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/data
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:30:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:30:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.975533  2453 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:30:08.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.977520  2453 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:30:08.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.978127  2453 kserver.cc:163] Server-wide thread pool size limit: 3276
18:30:08.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.985180  2453 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:30:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:08.992765  2485 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:30:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.001734  2453 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:30:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.002022  2453 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.002s
18:30:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.002341  2453 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:30:09.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.009114  2453 ts_tablet_manager.cc:616] Registered 1 tablets
18:30:09.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.009347  2453 ts_tablet_manager.cc:595] Time spent register tablets: real 0.007s	user 0.006s	sys 0.001s
18:30:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.009847  2485 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap starting.
18:30:09.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.071818  2485 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Log is configured to *not* fsync() on all Append() calls
18:30:09.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.142727  2485 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:30:09.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.143726  2485 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap complete.
18:30:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.145249  2485 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Time spent bootstrapping tablet: real 0.136s	user 0.092s	sys 0.039s
18:30:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.163120  2485 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.167258  2485 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9df350dc58374330a22867bfa403ed8d, State: Initialized, Role: FOLLOWER
18:30:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.168730  2485 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:09.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.173539  2485 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Time spent starting tablet: real 0.028s	user 0.023s	sys 0.005s
18:30:09.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.203745  2453 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.66:44881
18:30:09.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.204030  2595 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.66:44881 every 8 connection(s)
18:30:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.206489  2453 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:30:09.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.211216  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 2453
18:30:09.212 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:09.253 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:30:09.256 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.16.65:45335
18:30:09.257 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:30:09.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.257555  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 2095
18:30:09.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.280032  2602 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:30:09.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.280602  2602 heartbeater.cc:461] Registering TS with master...
18:30:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.281932  2602 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:30:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.282102  2597 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:30:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.282464  2597 heartbeater.cc:461] Registering TS with master...
18:30:09.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.283394  2597 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:30:09.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.285230  2596 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:30:09.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.285575  2596 heartbeater.cc:461] Registering TS with master...
18:30:09.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.286396  2596 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:30:09.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.286774  1134 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:09.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.287533  1209 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:09.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.289745  1287 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:09.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:09.292272  1287 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.66:55793
18:30:09.299 [DEBUG - kudu-nio-7] (Connection.java:239) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] handling channelInactive
18:30:09.299 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state READY due to: connection closed
18:30:09.299 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:09.300 [DEBUG - Time-limited test] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:09.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.302 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.308 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:09.308 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:09.309 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1577591863(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6a0a9e45 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e492ced -> wakeup thread Time-limited test)) after lookup
18:30:09.309 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:09.310 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.310 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.310 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.328 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:09.328 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:09.328 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [9ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [10ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [10ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1577591863(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6a0a9e45 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e492ced -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424))) after lookup
18:30:09.329 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:09.331 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.331 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.331 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.368 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:09.368 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:09.368 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=68), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [9ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [10ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [10ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [31ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [31ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1577591863(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6a0a9e45 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e492ced -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157))) after lookup
18:30:09.369 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:09.370 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.370 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.370 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.428 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:09.429 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:09.429 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=129), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [9ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [10ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [10ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [31ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [31ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [68ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [70ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [70ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [124ms] refreshing cache from master, [125ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [128ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1577591863(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6a0a9e45 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157) -> (continuation of Deferred@568268797 after retry RPC@1964314862), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e492ced -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157) -> (continuation of Deferred@568268797 after retry RPC@1964314862))) after lookup
18:30:09.429 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:09.430 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.431 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.431 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.488 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:09.489 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:09.489 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=189), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [9ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [10ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [10ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [31ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [31ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [68ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [70ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [70ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [124ms] refreshing cache from master, [125ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [128ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [129ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [131ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [131ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [184ms] refreshing cache from master, [185ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [188ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1577591863(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6a0a9e45 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157) -> (continuation of Deferred@568268797 after retry RPC@1964314862) -> (continuation of Deferred@1597527418 after retry RPC@49785875), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e492ced -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157) -> (continuation of Deferred@568268797 after retry RPC@1964314862) -> (continuation of Deferred@1597527418 after retry RPC@49785875))) after lookup
18:30:09.490 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:09.490 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.490 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:09.491 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:10.108 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:10.108 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:10.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=808), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [9ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [10ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [10ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [31ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [31ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [68ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [70ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [70ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [124ms] refreshing cache from master, [125ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [128ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [129ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [131ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [131ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [184ms] refreshing cache from master, [185ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [188ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [189ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [191ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [191ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [804ms] refreshing cache from master, [804ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [808ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1577591863(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6a0a9e45 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157) -> (continuation of Deferred@568268797 after retry RPC@1964314862) -> (continuation of Deferred@1597527418 after retry RPC@49785875) -> (continuation of Deferred@258657007 after retry RPC@809658405), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e492ced -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157) -> (continuation of Deferred@568268797 after retry RPC@1964314862) -> (continuation of Deferred@1597527418 after retry RPC@49785875) -> (continuation of Deferred@258657007 after retry RPC@809658405))) after lookup
18:30:10.109 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:10.111 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:10.111 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:10.111 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:10.249 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:10.249 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:10.250 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=950), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [9ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [10ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [10ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [31ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [31ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [68ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [70ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [70ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [124ms] refreshing cache from master, [125ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [128ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [129ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [131ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [131ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [184ms] refreshing cache from master, [185ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [188ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [189ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [191ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [191ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [804ms] refreshing cache from master, [804ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [808ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [809ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [811ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [811ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [945ms] refreshing cache from master, [945ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [949ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1577591863(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6a0a9e45 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157) -> (continuation of Deferred@568268797 after retry RPC@1964314862) -> (continuation of Deferred@1597527418 after retry RPC@49785875) -> (continuation of Deferred@258657007 after retry RPC@809658405) -> (continuation of Deferred@134198628 after retry RPC@610264830), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e492ced -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157) -> (continuation of Deferred@568268797 after retry RPC@1964314862) -> (continuation of Deferred@1597527418 after retry RPC@49785875) -> (continuation of Deferred@258657007 after retry RPC@809658405) -> (continuation of Deferred@134198628 after retry RPC@610264830))) after lookup
18:30:10.250 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:10.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:10.251 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:10.252 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.296226  2596 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:30:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.471306  2607 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:30:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.471745  2607 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.474036  2607 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:10.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:10.479336  2480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:30:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:10.483655  2480 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:30:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.486342  2367 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1" is_pre_election: true
18:30:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.487848  2481 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d; no voters: a8fdfc4241d34c848f5eb3275d07e1f6, bacacb109f8247dcb77a540bd9e605a1
18:30:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.488621  2607 raft_consensus.cc:2749] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
18:30:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.540616  2612 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader a8fdfc4241d34c848f5eb3275d07e1f6)
18:30:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.541030  2612 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.543367  2612 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:10.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:10.547865  2299 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:30:10.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:10.551877  2299 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:30:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.555576  2550 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "bacacb109f8247dcb77a540bd9e605a1" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "9df350dc58374330a22867bfa403ed8d" is_pre_election: true
18:30:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.556257  2550 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bacacb109f8247dcb77a540bd9e605a1 in term 4.
18:30:10.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.557654  2300 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, bacacb109f8247dcb77a540bd9e605a1; no voters: a8fdfc4241d34c848f5eb3275d07e1f6
18:30:10.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.558485  2612 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 5 FOLLOWER]: Leader pre-election won for term 6
18:30:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.558826  2612 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 5 FOLLOWER]: Starting leader election (detected failure of leader a8fdfc4241d34c848f5eb3275d07e1f6)
18:30:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.559343  2612 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 5 FOLLOWER]: Advancing to term 6
18:30:10.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.563737  2612 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.565228  2612 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 6 election: Requested vote from peers 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:10.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.566075  2550 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "bacacb109f8247dcb77a540bd9e605a1" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "9df350dc58374330a22867bfa403ed8d"
18:30:10.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.566634  2550 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 4 FOLLOWER]: Advancing to term 6
18:30:10.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:10.569129  2299 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:30:10.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.574067  2550 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bacacb109f8247dcb77a540bd9e605a1 in term 6.
18:30:10.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.575040  2300 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, bacacb109f8247dcb77a540bd9e605a1; no voters: a8fdfc4241d34c848f5eb3275d07e1f6
18:30:10.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.575656  2612 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 6 FOLLOWER]: Leader election won for term 6
18:30:10.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.577320  2612 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 6 LEADER]: Becoming Leader. State: Replica: bacacb109f8247dcb77a540bd9e605a1, State: Running, Role: LEADER
18:30:10.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.578563  2612 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:10.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.587999  1287 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 reported cstate change: term changed from 5 to 6, leader changed from a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65) to bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67). New cstate: current_term: 6 leader_uuid: "bacacb109f8247dcb77a540bd9e605a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: UNKNOWN } } }
18:30:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.992895  2550 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 6 FOLLOWER]: Refusing update from remote peer bacacb109f8247dcb77a540bd9e605a1: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
18:30:10.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:10.994648  2612 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
18:30:11.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:11.030831  2299 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 -> Peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Couldn't send request to peer a8fdfc4241d34c848f5eb3275d07e1f6. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:30:13.287 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:13.288 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:13.288 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=16, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=3988), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [9ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [10ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [10ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [29ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [31ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [31ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [68ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [70ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [70ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [124ms] refreshing cache from master, [125ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [128ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [129ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [131ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [131ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [184ms] refreshing cache from master, [185ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [188ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [189ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [191ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [191ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [804ms] refreshing cache from master, [804ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [808ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [809ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [811ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [811ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [945ms] refreshing cache from master, [945ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [949ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [950ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [951ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [952ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [3984ms] refreshing cache from master, [3984ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [3987ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1577591863(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@6a0a9e45 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157) -> (continuation of Deferred@568268797 after retry RPC@1964314862) -> (continuation of Deferred@1597527418 after retry RPC@49785875) -> (continuation of Deferred@258657007 after retry RPC@809658405) -> (continuation of Deferred@134198628 after retry RPC@610264830) -> (continuation of Deferred@2078258654 after retry RPC@544003149), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e492ced -> wakeup thread Time-limited test -> (continuation of Deferred@150923222 after retry RPC@1326947424) -> (continuation of Deferred@447794847 after retry RPC@1773647157) -> (continuation of Deferred@568268797 after retry RPC@1964314862) -> (continuation of Deferred@1597527418 after retry RPC@49785875) -> (continuation of Deferred@258657007 after retry RPC@809658405) -> (continuation of Deferred@134198628 after retry RPC@610264830) -> (continuation of Deferred@2078258654 after retry RPC@544003149))) after lookup
18:30:13.289 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:13.290 [DEBUG - kudu-nio-0] (Connection.java:757) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] Successfully connected to peer
18:30:13.319 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd4fe2e3d, L:/127.0.0.1:57340 - R:/127.1.16.67:37285] using TOKEN/null
18:30:13.374 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.16.65:45335
18:30:13.375 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:13.377897  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/logs
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.65:45335
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.65
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44059
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.65
18:30:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:13.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:13.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:13.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:30:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:13.415280  2299 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 -> Peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Couldn't send request to peer a8fdfc4241d34c848f5eb3275d07e1f6. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
18:30:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:13.694975  2653 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:30:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:13.695510  2653 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:30:13.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:13.696161  2653 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:30:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:13.728271  2653 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:30:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:13.729271  2653 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.65
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:13.767771  2653 tablet_server_runner.cc:78] Tablet server non-default flags:
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.65:45335
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.65
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44059
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.65
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/logs
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:30:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:30:13.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:30:13.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:30:13.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:13.769603  2653 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:30:13.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:13.771548  2653 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:30:13.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:13.787561  2661 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:13.803030  2664 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:13.789362  2662 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:15.104312  2663 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1315 milliseconds
18:30:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.104383  2653 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:30:15.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.109397  2653 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:30:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.111773  2653 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:30:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.113190  2653 hybrid_clock.cc:648] HybridClock initialized: now 1783535415113154 us; error 36 us; skew 500 ppm
18:30:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.114151  2653 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:30:15.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.120805  2653 webserver.cc:534] Webserver started at http://127.1.16.65:44059/ using document root <none> and password file <none>
18:30:15.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.121758  2653 fs_manager.cc:362] Metadata directory not provided
18:30:15.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.122008  2653 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:30:15.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.129997  2653 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.004s	sys 0.000s
18:30:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.135017  2671 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:30:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.136277  2653 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
18:30:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.136696  2653 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8fdfc4241d34c848f5eb3275d07e1f6"
18:30:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:42 on dist-test-slave-l44q"
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.138646  2653 fs_report.cc:389] FS layout report
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/data
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:30:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:30:15.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.226694  2653 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:30:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.228789  2653 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:30:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.229382  2653 kserver.cc:163] Server-wide thread pool size limit: 3276
18:30:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.233085  2653 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:30:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.240828  2680 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:30:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.249732  2653 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:30:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.249936  2653 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
18:30:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.250140  2653 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:30:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.255120  2653 ts_tablet_manager.cc:616] Registered 1 tablets
18:30:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.255299  2653 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
18:30:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.255875  2680 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Bootstrap starting.
18:30:15.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.314881  2680 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Log is configured to *not* fsync() on all Append() calls
18:30:15.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.391379  2680 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:30:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.392555  2680 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Bootstrap complete.
18:30:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.394052  2680 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Time spent bootstrapping tablet: real 0.139s	user 0.093s	sys 0.042s
18:30:15.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.414494  2680 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:15.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.417048  2680 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8fdfc4241d34c848f5eb3275d07e1f6, State: Initialized, Role: FOLLOWER
18:30:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.418394  2680 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:15.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.423221  2680 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Time spent starting tablet: real 0.029s	user 0.022s	sys 0.009s
18:30:15.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.436656  2653 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.65:45335
18:30:15.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.436765  2789 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.65:45335 every 8 connection(s)
18:30:15.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.439179  2653 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:30:15.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.449143  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 2653
18:30:15.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:15.494 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:30:15.494 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.16.67:37285
18:30:15.495 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:30:15.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.495646  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 2276
18:30:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.499114  2790 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:30:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.499640  2790 heartbeater.cc:461] Registering TS with master...
18:30:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.501423  2790 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:30:15.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.505519  1287 ts_manager.cc:194] Re-registered known tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:15.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.505751  2792 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:30:15.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.506139  2792 heartbeater.cc:461] Registering TS with master...
18:30:15.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.506232  2791 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:30:15.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.506629  2791 heartbeater.cc:461] Registering TS with master...
18:30:15.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.507156  2792 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:30:15.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.507519  2791 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:30:15.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.508810  1287 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.65:40901
18:30:15.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.511497  1134 ts_manager.cc:194] Re-registered known tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:15.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:15.511583  1209 ts_manager.cc:194] Re-registered known tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:15.529 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:15.530 [DEBUG - kudu-nio-0] (Connection.java:239) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] handling channelInactive
18:30:15.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state READY due to: connection closed
18:30:15.545 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:15.546 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.547 [DEBUG - kudu-nio-1] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.547 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.567 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:15.568 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:15.568 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=3, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [17ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [17ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1046232266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@8955849 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@351bab31 -> wakeup thread Time-limited test)) after lookup
18:30:15.568 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:15.569 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.570 [DEBUG - kudu-nio-2] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.570 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.588 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:15.589 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:15.589 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=5, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [17ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [17ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [38ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [40ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [40ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [58ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1046232266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@8955849 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@90032565 after retry RPC@1844946956), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@351bab31 -> wakeup thread Time-limited test -> (continuation of Deferred@90032565 after retry RPC@1844946956))) after lookup
18:30:15.590 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:15.591 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.591 [DEBUG - kudu-nio-3] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.591 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.668 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:15.669 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:15.669 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=7, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=139), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [17ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [17ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [38ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [40ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [40ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [58ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [59ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [61ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [61ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [134ms] refreshing cache from master, [134ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [138ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1046232266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@8955849 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@90032565 after retry RPC@1844946956) -> (continuation of Deferred@2010468635 after retry RPC@657600549), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@351bab31 -> wakeup thread Time-limited test -> (continuation of Deferred@90032565 after retry RPC@1844946956) -> (continuation of Deferred@2010468635 after retry RPC@657600549))) after lookup
18:30:15.670 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:15.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.671 [DEBUG - kudu-nio-4] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.671 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.828 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:15.828 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:15.829 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=9, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=299), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [17ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [17ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [38ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [40ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [40ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [58ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [59ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [61ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [61ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [134ms] refreshing cache from master, [134ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [138ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [139ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [141ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [141ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [294ms] refreshing cache from master, [294ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [298ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1046232266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@8955849 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@90032565 after retry RPC@1844946956) -> (continuation of Deferred@2010468635 after retry RPC@657600549) -> (continuation of Deferred@1144951306 after retry RPC@1924870576), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@351bab31 -> wakeup thread Time-limited test -> (continuation of Deferred@90032565 after retry RPC@1844946956) -> (continuation of Deferred@2010468635 after retry RPC@657600549) -> (continuation of Deferred@1144951306 after retry RPC@1924870576))) after lookup
18:30:15.829 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:15.830 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.831 [DEBUG - kudu-nio-5] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.831 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.868 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:15.869 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:15.869 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=11, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=339), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [17ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [17ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [38ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [40ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [40ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [58ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [59ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [61ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [61ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [134ms] refreshing cache from master, [134ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [138ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [139ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [141ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [141ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [294ms] refreshing cache from master, [294ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [298ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [299ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [301ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [301ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [334ms] refreshing cache from master, [335ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [338ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1046232266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@8955849 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@90032565 after retry RPC@1844946956) -> (continuation of Deferred@2010468635 after retry RPC@657600549) -> (continuation of Deferred@1144951306 after retry RPC@1924870576) -> (continuation of Deferred@1695988484 after retry RPC@1317744807), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@351bab31 -> wakeup thread Time-limited test -> (continuation of Deferred@90032565 after retry RPC@1844946956) -> (continuation of Deferred@2010468635 after retry RPC@657600549) -> (continuation of Deferred@1144951306 after retry RPC@1924870576) -> (continuation of Deferred@1695988484 after retry RPC@1317744807))) after lookup
18:30:15.869 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:15.870 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.870 [DEBUG - kudu-nio-6] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:15.870 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.513805  2790 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:30:16.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.859798  2802 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:30:16.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.860206  2802 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.862617  2802 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:16.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:16.867134  2677 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111)
18:30:16.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:16.871562  2677 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111)
18:30:16.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.874644  2550 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "9df350dc58374330a22867bfa403ed8d" is_pre_election: true
18:30:16.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.876034  2677 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a8fdfc4241d34c848f5eb3275d07e1f6; no voters: 9df350dc58374330a22867bfa403ed8d, bacacb109f8247dcb77a540bd9e605a1
18:30:16.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.876885  2802 raft_consensus.cc:2749] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
18:30:16.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.962615  2807 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 6 FOLLOWER]: Starting pre-election (detected failure of leader bacacb109f8247dcb77a540bd9e605a1)
18:30:16.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.963053  2807 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:16.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.964697  2807 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:16.967356  2481 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111)
18:30:16.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:16.970978  2481 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111)
18:30:16.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.977211  2743 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" is_pre_election: true
18:30:16.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.977967  2743 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9df350dc58374330a22867bfa403ed8d in term 5.
18:30:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.979346  2480 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, a8fdfc4241d34c848f5eb3275d07e1f6; no voters: bacacb109f8247dcb77a540bd9e605a1
18:30:16.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.980032  2807 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 6 FOLLOWER]: Leader pre-election won for term 7
18:30:16.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.980443  2807 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 6 FOLLOWER]: Starting leader election (detected failure of leader bacacb109f8247dcb77a540bd9e605a1)
18:30:16.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.980772  2807 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 6 FOLLOWER]: Advancing to term 7
18:30:16.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.985236  2807 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:16.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.987042  2807 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 7 election: Requested vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.988031  2743 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6"
18:30:16.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.988495  2743 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 5 FOLLOWER]: Advancing to term 7
18:30:16.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:16.991119  2481 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111)
18:30:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.995325  2743 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9df350dc58374330a22867bfa403ed8d in term 7.
18:30:16.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.996162  2480 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, a8fdfc4241d34c848f5eb3275d07e1f6; no voters: bacacb109f8247dcb77a540bd9e605a1
18:30:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.996848  2807 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 7 FOLLOWER]: Leader election won for term 7
18:30:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.997967  2807 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 7 LEADER]: Becoming Leader. State: Replica: 9df350dc58374330a22867bfa403ed8d, State: Running, Role: LEADER
18:30:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:16.998692  2807 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:17.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:17.007963  1287 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d reported cstate change: term changed from 6 to 7, leader changed from bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67) to 9df350dc58374330a22867bfa403ed8d (127.1.16.66). New cstate: current_term: 7 leader_uuid: "9df350dc58374330a22867bfa403ed8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: UNKNOWN } } }
18:30:17.069 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:17.069 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:17.069 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=13, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1539), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [17ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [17ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [38ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [40ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [40ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [58ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [59ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [61ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [61ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [134ms] refreshing cache from master, [134ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [138ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [139ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [141ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [141ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [294ms] refreshing cache from master, [294ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [298ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [299ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [301ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [301ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [334ms] refreshing cache from master, [335ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [338ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [339ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [340ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [341ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [1534ms] refreshing cache from master, [1535ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1539ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1046232266(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@8955849 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@90032565 after retry RPC@1844946956) -> (continuation of Deferred@2010468635 after retry RPC@657600549) -> (continuation of Deferred@1144951306 after retry RPC@1924870576) -> (continuation of Deferred@1695988484 after retry RPC@1317744807) -> (continuation of Deferred@806195458 after retry RPC@664048044), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@351bab31 -> wakeup thread Time-limited test -> (continuation of Deferred@90032565 after retry RPC@1844946956) -> (continuation of Deferred@2010468635 after retry RPC@657600549) -> (continuation of Deferred@1144951306 after retry RPC@1924870576) -> (continuation of Deferred@1695988484 after retry RPC@1317744807) -> (continuation of Deferred@806195458 after retry RPC@664048044))) after lookup
18:30:17.070 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:17.071 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] Successfully connected to peer
18:30:17.090 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x8f51b6ef, L:/127.0.0.1:43114 - R:/127.1.16.66:44881] using TOKEN/null
18:30:17.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:17.099237  2743 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 7 FOLLOWER]: Refusing update from remote peer 9df350dc58374330a22867bfa403ed8d: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 26. (index mismatch)
18:30:17.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:17.100924  2812 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
18:30:17.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:17.101604  2481 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d -> Peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Couldn't send request to peer bacacb109f8247dcb77a540bd9e605a1. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:30:17.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:17.126147  2824 mvcc.cc:204] Tried to move back new op lower bound from 7305361068424286208 to 7305361068035280896. Current Snapshot: MvccSnapshot[applied={T|T < 7305361068424286208}]
18:30:17.167 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.16.67:37285
18:30:17.168 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:17.169581  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/logs
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.67:37285
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.67
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37737
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.67
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:30:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:17.493924  2832 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:30:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:17.494372  2832 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:30:17.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:17.494825  2832 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:30:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:17.527591  2832 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:30:17.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:17.528708  2832 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.67
18:30:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:17.568259  2832 tablet_server_runner.cc:78] Tablet server non-default flags:
18:30:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.67:37285
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.67
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37737
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.67
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/logs
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:30:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:30:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:17.570019  2832 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:30:17.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:17.571628  2832 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:30:17.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:17.587086  2841 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:18.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:17.588603  2842 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:18.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:17.590811  2844 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:18.854527  2843 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1264 milliseconds
18:30:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.854681  2832 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:30:18.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.856045  2832 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:30:18.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.858942  2832 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:30:18.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.860460  2832 hybrid_clock.cc:648] HybridClock initialized: now 1783535418860398 us; error 66 us; skew 500 ppm
18:30:18.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.861232  2832 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:30:18.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.867877  2832 webserver.cc:534] Webserver started at http://127.1.16.67:37737/ using document root <none> and password file <none>
18:30:18.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.868841  2832 fs_manager.cc:362] Metadata directory not provided
18:30:18.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.869024  2832 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:30:18.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.877277  2832 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.007s	sys 0.000s
18:30:18.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.881681  2856 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:30:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.882670  2832 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.001s
18:30:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.883008  2832 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:47 on dist-test-slave-l44q"
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.884930  2832 fs_report.cc:389] FS layout report
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/data
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:30:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:30:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:30:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:30:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:30:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:30:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:30:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:30:18.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.934491  2832 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:30:18.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.935972  2832 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:30:18.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.936486  2832 kserver.cc:163] Server-wide thread pool size limit: 3276
18:30:18.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.939014  2832 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:30:18.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.944586  2863 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:30:18.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.952330  2832 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:30:18.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.952670  2832 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
18:30:18.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.953047  2832 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:30:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.957721  2832 ts_tablet_manager.cc:616] Registered 1 tablets
18:30:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.957923  2832 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
18:30:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:18.958252  2863 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Bootstrap starting.
18:30:19.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.032661  2863 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Log is configured to *not* fsync() on all Append() calls
18:30:19.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.143560  2832 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.67:37285
18:30:19.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.143631  2863 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:30:19.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.143693  2970 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.67:37285 every 8 connection(s)
18:30:19.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.144711  2863 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Bootstrap complete.
18:30:19.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.146543  2863 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Time spent bootstrapping tablet: real 0.189s	user 0.136s	sys 0.047s
18:30:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.147446  2832 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:30:19.153 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:19.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.151415  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 2832
18:30:19.180 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:30:19.180 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.16.66:44881
18:30:19.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:30:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.181553  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 2453
18:30:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.225395  2975 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:30:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.225859  2975 heartbeater.cc:461] Registering TS with master...
18:30:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.226464  2971 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:30:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.226825  2971 heartbeater.cc:461] Registering TS with master...
18:30:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.227236  2975 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:30:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.227799  2971 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:30:19.230 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] handling channelInactive
18:30:19.230 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state READY due to: connection closed
18:30:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:19.234109  2974 negotiation.cc:336] Failed RPC negotiation. Trace:
18:30:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:19.164378 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.1.16.66:54635 (local address 127.1.16.67:37285)
18:30:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:19.183688 (+ 19310us) server_negotiation.cc:207] Beginning negotiation
18:30:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:19.183729 (+    41us) server_negotiation.cc:400] Waiting for connection header
18:30:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:19.183924 (+   195us) server_negotiation.cc:408] Connection header received
18:30:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:19.199187 (+ 15263us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
18:30:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:19.199246 (+    59us) server_negotiation.cc:462] Received NEGOTIATE request from client
18:30:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:19.202222 (+  2976us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
18:30:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:19.202451 (+   229us) server_negotiation.cc:227] Negotiated authn=SASL
18:30:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:19.230071 (+ 27620us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.16.66:54635: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
18:30:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":17523,"thread_start_us":11889,"threads_started":1}
18:30:19.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:19.238 [DEBUG - Time-limited test] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:19.240 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.240 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.240 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.241844  1208 ts_manager.cc:194] Re-registered known tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:30:19.249 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:19.249 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:19.249 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=11), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1278351504(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c1ef4ce -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58e96f6a -> wakeup thread Time-limited test)) after lookup
18:30:19.250 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:19.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.252 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.235275  2863 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:19.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.253224  2863 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: bacacb109f8247dcb77a540bd9e605a1, State: Initialized, Role: FOLLOWER
18:30:19.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.255046  2863 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.256770  2980 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:30:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.257155  2980 heartbeater.cc:461] Registering TS with master...
18:30:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.258313  2980 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:30:19.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.260730  1287 ts_manager.cc:194] Re-registered known tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:30:19.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.261824  1134 ts_manager.cc:194] Re-registered known tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:30:19.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.262539  2863 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Time spent starting tablet: real 0.116s	user 0.022s	sys 0.015s
18:30:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.263844  1287 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.67:51351
18:30:19.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:19.267504  2971 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:30:19.269 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:19.270 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:19.270 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [11ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [14ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [14ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1278351504(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c1ef4ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58e96f6a -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015))) after lookup
18:30:19.271 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:19.271 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.272 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.272 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.307 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:19.308 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:19.308 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [11ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [14ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [14ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [32ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [34ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1278351504(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c1ef4ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58e96f6a -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953))) after lookup
18:30:19.308 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:19.309 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.309 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.309 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.327 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:19.327 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:19.327 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=89), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [11ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [14ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [14ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [32ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [34ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [70ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [71ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [71ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [86ms] refreshing cache from master, [86ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [89ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1278351504(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c1ef4ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953) -> (continuation of Deferred@1591903128 after retry RPC@991334685), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58e96f6a -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953) -> (continuation of Deferred@1591903128 after retry RPC@991334685))) after lookup
18:30:19.328 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:19.329 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.329 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.329 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.388 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:19.388 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:19.388 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=150), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [11ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [14ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [14ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [32ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [34ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [70ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [71ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [71ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [86ms] refreshing cache from master, [86ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [89ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [90ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [91ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [91ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [146ms] refreshing cache from master, [147ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [150ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1278351504(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c1ef4ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953) -> (continuation of Deferred@1591903128 after retry RPC@991334685) -> (continuation of Deferred@1363869397 after retry RPC@1583023000), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58e96f6a -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953) -> (continuation of Deferred@1591903128 after retry RPC@991334685) -> (continuation of Deferred@1363869397 after retry RPC@1583023000))) after lookup
18:30:19.389 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:19.390 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.390 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:19.390 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:20.389 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:20.389 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:20.390 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1151), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [11ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [14ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [14ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [32ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [34ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [70ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [71ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [71ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [86ms] refreshing cache from master, [86ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [89ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [90ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [91ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [91ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [146ms] refreshing cache from master, [147ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [150ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [151ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [152ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [152ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1146ms] refreshing cache from master, [1146ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1151ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1278351504(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c1ef4ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953) -> (continuation of Deferred@1591903128 after retry RPC@991334685) -> (continuation of Deferred@1363869397 after retry RPC@1583023000) -> (continuation of Deferred@1777252937 after retry RPC@1865364271), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58e96f6a -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953) -> (continuation of Deferred@1591903128 after retry RPC@991334685) -> (continuation of Deferred@1363869397 after retry RPC@1583023000) -> (continuation of Deferred@1777252937 after retry RPC@1865364271))) after lookup
18:30:20.391 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:20.392 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:20.392 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:20.392 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:20.427 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:20.428 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:20.428 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1190), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [11ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [14ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [14ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [32ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [34ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [70ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [71ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [71ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [86ms] refreshing cache from master, [86ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [89ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [90ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [91ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [91ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [146ms] refreshing cache from master, [147ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [150ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [151ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [152ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [152ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1146ms] refreshing cache from master, [1146ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1151ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [1152ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [1154ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1154ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1186ms] refreshing cache from master, [1186ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1189ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1278351504(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c1ef4ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953) -> (continuation of Deferred@1591903128 after retry RPC@991334685) -> (continuation of Deferred@1363869397 after retry RPC@1583023000) -> (continuation of Deferred@1777252937 after retry RPC@1865364271) -> (continuation of Deferred@998043411 after retry RPC@761005336), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58e96f6a -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953) -> (continuation of Deferred@1591903128 after retry RPC@991334685) -> (continuation of Deferred@1363869397 after retry RPC@1583023000) -> (continuation of Deferred@1777252937 after retry RPC@1865364271) -> (continuation of Deferred@998043411 after retry RPC@761005336))) after lookup
18:30:20.429 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:20.430 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:20.430 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:20.430 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:20.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.478334  2984 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 9df350dc58374330a22867bfa403ed8d)
18:30:20.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.478736  2984 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.480162  2984 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:20.487414  2677 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:30:20.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.492260  2926 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1" is_pre_election: true
18:30:20.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.492966  2926 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a8fdfc4241d34c848f5eb3275d07e1f6 in term 6.
18:30:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.494083  2677 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a8fdfc4241d34c848f5eb3275d07e1f6, bacacb109f8247dcb77a540bd9e605a1; no voters: 9df350dc58374330a22867bfa403ed8d
18:30:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.494666  2984 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 7 FOLLOWER]: Leader pre-election won for term 8
18:30:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.494966  2984 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 9df350dc58374330a22867bfa403ed8d)
18:30:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.495297  2984 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 7 FOLLOWER]: Advancing to term 8
18:30:20.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.499722  2984 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:20.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.501298  2984 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 8 election: Requested vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.501987  2926 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:30:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.502364  2926 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 6 FOLLOWER]: Advancing to term 8
18:30:20.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:20.504967  2677 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:30:20.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.508778  2926 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a8fdfc4241d34c848f5eb3275d07e1f6 in term 8.
18:30:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.509665  2677 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a8fdfc4241d34c848f5eb3275d07e1f6, bacacb109f8247dcb77a540bd9e605a1; no voters: 9df350dc58374330a22867bfa403ed8d
18:30:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.510272  2984 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 8 FOLLOWER]: Leader election won for term 8
18:30:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.511402  2984 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 8 LEADER]: Becoming Leader. State: Replica: a8fdfc4241d34c848f5eb3275d07e1f6, State: Running, Role: LEADER
18:30:20.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.512284  2984 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:20.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.521011  1288 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 reported cstate change: term changed from 7 to 8, leader changed from 9df350dc58374330a22867bfa403ed8d (127.1.16.66) to a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65). New cstate: current_term: 8 leader_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: HEALTHY } } }
18:30:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.910995  2926 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 8 FOLLOWER]: Refusing update from remote peer a8fdfc4241d34c848f5eb3275d07e1f6: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
18:30:20.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:20.912580  2984 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
18:30:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:20.943446  2677 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 -> Peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Couldn't send request to peer 9df350dc58374330a22867bfa403ed8d. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:30:21.970 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:21.971 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:21.971 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=16, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2733), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [2ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [11ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [14ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [14ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [32ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [34ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [34ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [70ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [71ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [71ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [86ms] refreshing cache from master, [86ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [89ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [90ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [91ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [91ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [146ms] refreshing cache from master, [147ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [150ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [151ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [152ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [152ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1146ms] refreshing cache from master, [1146ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1151ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [1152ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [1154ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1154ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1186ms] refreshing cache from master, [1186ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1189ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [1191ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [1192ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1192ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [2727ms] refreshing cache from master, [2728ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [2732ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1278351504(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@5c1ef4ce -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953) -> (continuation of Deferred@1591903128 after retry RPC@991334685) -> (continuation of Deferred@1363869397 after retry RPC@1583023000) -> (continuation of Deferred@1777252937 after retry RPC@1865364271) -> (continuation of Deferred@998043411 after retry RPC@761005336) -> (continuation of Deferred@1062718340 after retry RPC@1928228275), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58e96f6a -> wakeup thread Time-limited test -> (continuation of Deferred@2135293682 after retry RPC@1824808015) -> (continuation of Deferred@872092757 after retry RPC@2106789953) -> (continuation of Deferred@1591903128 after retry RPC@991334685) -> (continuation of Deferred@1363869397 after retry RPC@1583023000) -> (continuation of Deferred@1777252937 after retry RPC@1865364271) -> (continuation of Deferred@998043411 after retry RPC@761005336) -> (continuation of Deferred@1062718340 after retry RPC@1928228275))) after lookup
18:30:21.976 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:21.980 [DEBUG - kudu-nio-0] (Connection.java:757) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] Successfully connected to peer
18:30:22.003 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa1b6ebc2, L:/127.0.0.1:46974 - R:/127.1.16.65:45335] using TOKEN/null
18:30:22.049 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.16.66:44881
18:30:22.050 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:30:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:22.051256  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data
18:30:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:30:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:30:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:30:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/logs
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.66:44881
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.66
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36159
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.66
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:30:22.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:22.179973  2677 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111) [suppressed 4 similar messages]
18:30:22.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:22.358705  3019 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:30:22.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:22.359177  3019 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:30:22.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:22.359618  3019 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:30:22.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:22.391999  3019 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:30:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:22.392865  3019 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.66
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:22.428990  3019 tablet_server_runner.cc:78] Tablet server non-default flags:
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.66:44881
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.66
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36159
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:30:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.66
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/logs
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:30:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:22.430227  3019 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:30:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:22.432065  3019 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:30:22.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:22.446314  3027 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:22.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:22.981777  2967 debug-util.cc:398] Leaking SignalData structure 0x7b08000ca6a0 after lost signal to thread 2835
18:30:22.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:22.983356  2967 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba080 after lost signal to thread 2970
18:30:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:23.478963  2677 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 -> Peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Couldn't send request to peer 9df350dc58374330a22867bfa403ed8d. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
18:30:23.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:22.447300  3028 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:23.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:23.797870  3030 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:23.800750  3029 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
18:30:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:23.801383  3019 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.354s	user 0.445s	sys 0.882s
18:30:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:23.801666  3019 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.354s	user 0.445s	sys 0.882s
18:30:23.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.801971  3019 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:30:23.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.803341  3019 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:30:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.829013  3019 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:30:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.830590  3019 hybrid_clock.cc:648] HybridClock initialized: now 1783535423830532 us; error 60 us; skew 500 ppm
18:30:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.831998  3019 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:30:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.869122  3019 webserver.cc:534] Webserver started at http://127.1.16.66:36159/ using document root <none> and password file <none>
18:30:23.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.870769  3019 fs_manager.cc:362] Metadata directory not provided
18:30:23.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.871240  3019 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:30:23.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.898404  3019 fs_manager.cc:714] Time spent opening directory manager: real 0.023s	user 0.004s	sys 0.019s
18:30:23.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.917299  3042 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:30:23.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.919272  3019 fs_manager.cc:730] Time spent opening block manager: real 0.018s	user 0.001s	sys 0.009s
18:30:23.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.919850  3019 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:23.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9df350dc58374330a22867bfa403ed8d"
18:30:23.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:45 on dist-test-slave-l44q"
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:23.922912  3019 fs_report.cc:389] FS layout report
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/data
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:30:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:30:23.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:30:23.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:30:23.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:30:23.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:30:24.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.018757  3019 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:30:24.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.021063  3019 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:30:24.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.021787  3019 kserver.cc:163] Server-wide thread pool size limit: 3276
18:30:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.035153  3019 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:30:24.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.042682  3049 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:30:24.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:24.043145  1165 debug-util.cc:398] Leaking SignalData structure 0x7b08000883a0 after lost signal to thread 1102
18:30:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:24.044045  1165 debug-util.cc:398] Leaking SignalData structure 0x7b080008dea0 after lost signal to thread 1168
18:30:24.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.055953  3019 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:30:24.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.056247  3019 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.015s	user 0.002s	sys 0.000s
18:30:24.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.056602  3019 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:30:24.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.063349  3019 ts_tablet_manager.cc:616] Registered 1 tablets
18:30:24.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.063589  3019 ts_tablet_manager.cc:595] Time spent register tablets: real 0.007s	user 0.004s	sys 0.000s
18:30:24.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.064301  3049 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap starting.
18:30:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.146394  3049 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Log is configured to *not* fsync() on all Append() calls
18:30:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.269479  3049 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:30:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.271652  3049 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap complete.
18:30:24.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.273823  3049 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Time spent bootstrapping tablet: real 0.210s	user 0.152s	sys 0.042s
18:30:24.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.301054  3049 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:24.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.304867  3049 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9df350dc58374330a22867bfa403ed8d, State: Initialized, Role: FOLLOWER
18:30:24.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.308817  3049 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:24.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.318956  3049 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Time spent starting tablet: real 0.044s	user 0.027s	sys 0.011s
18:30:24.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.876201  3019 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.66:44881
18:30:24.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.876335  3157 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.66:44881 every 8 connection(s)
18:30:24.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.880465  3019 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:30:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.888885  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 3019
18:30:24.891 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:24.959 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:30:24.960 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.16.65:45335
18:30:24.960 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:30:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.961031  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 2653
18:30:24.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.977016  3158 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:30:24.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.977507  3158 heartbeater.cc:461] Registering TS with master...
18:30:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.978820  3158 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:30:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.985162  1288 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.989701  1288 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.66:55491
18:30:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.995033  3160 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:30:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.995381  3160 heartbeater.cc:461] Registering TS with master...
18:30:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:24.996039  3160 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:30:25.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.000744  1209 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:25.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.000939  3163 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:30:25.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.001276  3163 heartbeater.cc:461] Registering TS with master...
18:30:25.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.001899  3163 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:30:25.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.005537  1134 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:25.009 [DEBUG - kudu-nio-0] (Connection.java:239) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] handling channelInactive
18:30:25.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state READY due to: connection closed
18:30:25.010 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:25.011 [DEBUG - Time-limited test] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:25.013 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.013 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.013 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.027 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:25.028 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:25.028 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@309007523(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@23b38026 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@728015bb -> wakeup thread Time-limited test)) after lookup
18:30:25.028 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:25.029 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.029 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.030 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:25.048 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:25.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [17ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [19ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [19ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@309007523(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@23b38026 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1403325945 after retry RPC@496997807), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@728015bb -> wakeup thread Time-limited test -> (continuation of Deferred@1403325945 after retry RPC@496997807))) after lookup
18:30:25.049 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:25.050 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.050 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.050 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.068 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:25.069 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:25.069 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=58), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [17ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [19ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [19ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [38ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [39ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [39ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [57ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@309007523(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@23b38026 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1403325945 after retry RPC@496997807) -> (continuation of Deferred@652089044 after retry RPC@1200479565), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@728015bb -> wakeup thread Time-limited test -> (continuation of Deferred@1403325945 after retry RPC@496997807) -> (continuation of Deferred@652089044 after retry RPC@1200479565))) after lookup
18:30:25.069 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:25.070 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.070 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.071 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.108 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:25.108 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:25.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=98), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [17ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [19ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [19ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [38ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [39ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [39ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [57ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [58ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [59ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [60ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [93ms] refreshing cache from master, [93ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [96ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@309007523(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@23b38026 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1403325945 after retry RPC@496997807) -> (continuation of Deferred@652089044 after retry RPC@1200479565) -> (continuation of Deferred@232389399 after retry RPC@605501524), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@728015bb -> wakeup thread Time-limited test -> (continuation of Deferred@1403325945 after retry RPC@496997807) -> (continuation of Deferred@652089044 after retry RPC@1200479565) -> (continuation of Deferred@232389399 after retry RPC@605501524))) after lookup
18:30:25.111 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:25.113 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.113 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.113 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.309 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:25.309 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:25.310 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=299), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [17ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [19ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [19ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [38ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [39ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [39ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [57ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [58ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [59ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [60ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [93ms] refreshing cache from master, [93ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [96ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [100ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [102ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [102ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [293ms] refreshing cache from master, [293ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [298ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@309007523(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@23b38026 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1403325945 after retry RPC@496997807) -> (continuation of Deferred@652089044 after retry RPC@1200479565) -> (continuation of Deferred@232389399 after retry RPC@605501524) -> (continuation of Deferred@19818348 after retry RPC@476457605), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@728015bb -> wakeup thread Time-limited test -> (continuation of Deferred@1403325945 after retry RPC@496997807) -> (continuation of Deferred@652089044 after retry RPC@1200479565) -> (continuation of Deferred@232389399 after retry RPC@605501524) -> (continuation of Deferred@19818348 after retry RPC@476457605))) after lookup
18:30:25.311 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:25.312 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.312 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state CONNECTING due to: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.312 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335
18:30:25.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.617372  3170 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:30:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.617866  3170 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.620534  3170 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:25.637892  3045 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:30:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:25.651633  3045 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:30:25.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.653004  2926 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1" is_pre_election: true
18:30:25.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.654484  3046 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d; no voters: a8fdfc4241d34c848f5eb3275d07e1f6, bacacb109f8247dcb77a540bd9e605a1
18:30:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.655208  3170 raft_consensus.cc:2749] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
18:30:25.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:25.995030  3158 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:30:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.218693  3175 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader a8fdfc4241d34c848f5eb3275d07e1f6)
18:30:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.219135  3175 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:26.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.221532  3175 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:26.235620  2859 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:30:26.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:26.242365  2859 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:30:26.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.243963  3113 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "bacacb109f8247dcb77a540bd9e605a1" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "9df350dc58374330a22867bfa403ed8d" is_pre_election: true
18:30:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.244760  3113 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bacacb109f8247dcb77a540bd9e605a1 in term 7.
18:30:26.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.246351  2860 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, bacacb109f8247dcb77a540bd9e605a1; no voters: a8fdfc4241d34c848f5eb3275d07e1f6
18:30:26.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.247188  3175 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 8 FOLLOWER]: Leader pre-election won for term 9
18:30:26.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.247483  3175 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 8 FOLLOWER]: Starting leader election (detected failure of leader a8fdfc4241d34c848f5eb3275d07e1f6)
18:30:26.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.247795  3175 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 8 FOLLOWER]: Advancing to term 9
18:30:26.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.252146  3175 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:26.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.253579  3175 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 9 election: Requested vote from peers 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:26.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.254592  3113 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "bacacb109f8247dcb77a540bd9e605a1" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "9df350dc58374330a22867bfa403ed8d"
18:30:26.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.255149  3113 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 7 FOLLOWER]: Advancing to term 9
18:30:26.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:26.257010  2859 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111)
18:30:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.265437  3113 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bacacb109f8247dcb77a540bd9e605a1 in term 9.
18:30:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.266637  2860 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, bacacb109f8247dcb77a540bd9e605a1; no voters: a8fdfc4241d34c848f5eb3275d07e1f6
18:30:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.267422  3175 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 9 FOLLOWER]: Leader election won for term 9
18:30:26.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.269272  3175 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 9 LEADER]: Becoming Leader. State: Replica: bacacb109f8247dcb77a540bd9e605a1, State: Running, Role: LEADER
18:30:26.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.270159  3175 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:26.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.279958  1288 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 reported cstate change: term changed from 8 to 9, leader changed from a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65) to bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67). New cstate: current_term: 9 leader_uuid: "bacacb109f8247dcb77a540bd9e605a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: UNKNOWN } } }
18:30:26.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.752628  3113 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 9 FOLLOWER]: Refusing update from remote peer bacacb109f8247dcb77a540bd9e605a1: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
18:30:26.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:26.754050  3175 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
18:30:26.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:26.786463  2859 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 -> Peer a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335): Couldn't send request to peer a8fdfc4241d34c848f5eb3275d07e1f6. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.65:45335: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:30:27.248 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:27.249 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:27.249 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2238), Traces: [0ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [2ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [17ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [19ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [19ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [38ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [39ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [39ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [57ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [58ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [59ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [60ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [93ms] refreshing cache from master, [93ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [96ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [100ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [102ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [102ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [293ms] refreshing cache from master, [293ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [298ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [300ms] sending RPC to server a8fdfc4241d34c848f5eb3275d07e1f6, [301ms] received response from server a8fdfc4241d34c848f5eb3275d07e1f6: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [301ms] delaying RPC due to: Network error: Failed to connect to peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335): Connection refused: /127.1.16.65:45335, [2233ms] refreshing cache from master, [2234ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [2237ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@309007523(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@23b38026 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1403325945 after retry RPC@496997807) -> (continuation of Deferred@652089044 after retry RPC@1200479565) -> (continuation of Deferred@232389399 after retry RPC@605501524) -> (continuation of Deferred@19818348 after retry RPC@476457605) -> (continuation of Deferred@655844478 after retry RPC@176701467), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@728015bb -> wakeup thread Time-limited test -> (continuation of Deferred@1403325945 after retry RPC@496997807) -> (continuation of Deferred@652089044 after retry RPC@1200479565) -> (continuation of Deferred@232389399 after retry RPC@605501524) -> (continuation of Deferred@19818348 after retry RPC@476457605) -> (continuation of Deferred@655844478 after retry RPC@176701467))) after lookup
18:30:27.250 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:27.251 [DEBUG - kudu-nio-7] (Connection.java:757) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] Successfully connected to peer
18:30:27.270 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x4f5613ad, L:/127.0.0.1:59442 - R:/127.1.16.67:37285] using TOKEN/null
18:30:27.317 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.16.65:45335
18:30:27.317 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:27.318856  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/logs
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:30:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.65:45335
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.65
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44059
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.65
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:30:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:27.647328  3208 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:30:27.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:27.647861  3208 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:30:27.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:27.648501  3208 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:30:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:27.680823  3208 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:30:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:27.681617  3208 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.65
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:27.741199  3208 tablet_server_runner.cc:78] Tablet server non-default flags:
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.65:45335
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.65
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44059
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.65
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/logs
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:30:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:27.742436  3208 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:30:27.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:27.743965  3208 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:30:27.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:27.755667  3215 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:29.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:27.758203  3216 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:29.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:29.070799  3218 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:29.073740  3217 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1312 milliseconds
18:30:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.073961  3208 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:30:29.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.076507  3208 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:30:29.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.080015  3208 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:30:29.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.081470  3208 hybrid_clock.cc:648] HybridClock initialized: now 1783535429081426 us; error 50 us; skew 500 ppm
18:30:29.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.082573  3208 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:30:29.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.090004  3208 webserver.cc:534] Webserver started at http://127.1.16.65:44059/ using document root <none> and password file <none>
18:30:29.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.091022  3208 fs_manager.cc:362] Metadata directory not provided
18:30:29.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.091274  3208 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:30:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.099427  3208 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.001s	sys 0.004s
18:30:29.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.104662  3227 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:30:29.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.105962  3208 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.003s
18:30:29.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.106326  3208 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:29.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8fdfc4241d34c848f5eb3275d07e1f6"
18:30:29.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:42 on dist-test-slave-l44q"
18:30:29.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.108294  3208 fs_report.cc:389] FS layout report
18:30:29.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/wal
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/data
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:30:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:30:29.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.155704  3208 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:30:29.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.157331  3208 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:30:29.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.157856  3208 kserver.cc:163] Server-wide thread pool size limit: 3276
18:30:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.160533  3208 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:30:29.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.166229  3234 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:30:29.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.174108  3208 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:30:29.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.174360  3208 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.000s	sys 0.002s
18:30:29.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.174710  3208 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:30:29.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.179540  3208 ts_tablet_manager.cc:616] Registered 1 tablets
18:30:29.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.179740  3208 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.001s	sys 0.001s
18:30:29.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.180037  3234 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Bootstrap starting.
18:30:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.253234  3234 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Log is configured to *not* fsync() on all Append() calls
18:30:29.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.349581  3208 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.65:45335
18:30:29.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.349709  3341 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.65:45335 every 8 connection(s)
18:30:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.353072  3208 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-0/data/info.pb
18:30:29.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.356940  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 3208
18:30:29.357 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:29.398 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:30:29.399 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.16.67:37285
18:30:29.399 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:30:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.400029  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 2832
18:30:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.423164  3344 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:30:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.423770  3344 heartbeater.cc:461] Registering TS with master...
18:30:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.425151  3344 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:30:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.430034  1209 ts_manager.cc:194] Re-registered known tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:29.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.434638  3234 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:30:29.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.435556  3348 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:30:29.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.435622  3234 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Bootstrap complete.
18:30:29.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.435956  3348 heartbeater.cc:461] Registering TS with master...
18:30:29.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.436975  3348 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:30:29.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.437546  3234 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Time spent bootstrapping tablet: real 0.258s	user 0.157s	sys 0.052s
18:30:29.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.441414  1133 ts_manager.cc:194] Re-registered known tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:29.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.442353  3342 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:30:29.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.442744  3342 heartbeater.cc:461] Registering TS with master...
18:30:29.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.443665  3342 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:30:29.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.446926  1287 ts_manager.cc:194] Re-registered known tserver with Master: a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.450348  1287 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.65:51403
18:30:29.452 [DEBUG - kudu-nio-7] (Connection.java:239) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] handling channelInactive
18:30:29.452 [DEBUG - kudu-nio-7] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state READY due to: connection closed
18:30:29.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:29.453919  3349 negotiation.cc:336] Failed RPC negotiation. Trace:
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:29.374981 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.1.16.67:38293 (local address 127.1.16.65:45335)
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:29.398753 (+ 23772us) server_negotiation.cc:207] Beginning negotiation
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:29.398779 (+    26us) server_negotiation.cc:400] Waiting for connection header
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:29.398849 (+    70us) server_negotiation.cc:408] Connection header received
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:29.399202 (+   353us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:29.399253 (+    51us) server_negotiation.cc:462] Received NEGOTIATE request from client
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:29.399690 (+   437us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:29.400189 (+   499us) server_negotiation.cc:227] Negotiated authn=SASL
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 18:30:29.452017 (+ 51828us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.16.67:38293: BlockingRecv error: recv got EOF from 127.1.16.67:38293 (error 108)
18:30:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":23108,"thread_start_us":26794,"threads_started":1}
18:30:29.455 [DEBUG - Time-limited test] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:29.456 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.456 [DEBUG - kudu-nio-0] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.456 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.457124  3234 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:29.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.460184  3234 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8fdfc4241d34c848f5eb3275d07e1f6, State: Initialized, Role: FOLLOWER
18:30:29.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.461133  3234 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:29.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.464208  3234 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6: Time spent starting tablet: real 0.026s	user 0.018s	sys 0.005s
18:30:29.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:29.465976  3342 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:30:29.473 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:29.473 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:29.474 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [3ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [3ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1240697359(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@53f8b967 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10c91bd4 -> wakeup thread Time-limited test)) after lookup
18:30:29.474 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:29.475 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.475 [DEBUG - kudu-nio-1] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.476 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.487 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:29.487 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:29.488 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [3ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [3ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [21ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [22ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [23ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [34ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1240697359(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@53f8b967 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10c91bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680))) after lookup
18:30:29.488 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:29.489 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.489 [DEBUG - kudu-nio-2] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.489 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.507 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:29.508 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:29.508 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=54), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [3ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [3ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [21ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [22ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [23ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [34ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [35ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [36ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [36ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [54ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1240697359(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@53f8b967 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680) -> (continuation of Deferred@176229339 after retry RPC@75630850), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10c91bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680) -> (continuation of Deferred@176229339 after retry RPC@75630850))) after lookup
18:30:29.508 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:29.509 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.510 [DEBUG - kudu-nio-3] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.510 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.647 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:29.647 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:29.648 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=194), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [3ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [3ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [21ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [22ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [23ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [34ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [35ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [36ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [36ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [54ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [55ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [57ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [57ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [194ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1240697359(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@53f8b967 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680) -> (continuation of Deferred@176229339 after retry RPC@75630850) -> (continuation of Deferred@1834273722 after retry RPC@2039088395), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10c91bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680) -> (continuation of Deferred@176229339 after retry RPC@75630850) -> (continuation of Deferred@1834273722 after retry RPC@2039088395))) after lookup
18:30:29.648 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:29.650 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.650 [DEBUG - kudu-nio-4] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.650 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.747 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:29.748 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:29.748 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=294), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [3ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [3ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [21ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [22ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [23ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [34ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [35ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [36ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [36ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [54ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [55ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [57ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [57ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [194ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [195ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [197ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [197ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [291ms] refreshing cache from master, [291ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [294ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1240697359(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@53f8b967 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680) -> (continuation of Deferred@176229339 after retry RPC@75630850) -> (continuation of Deferred@1834273722 after retry RPC@2039088395) -> (continuation of Deferred@838945944 after retry RPC@453682574), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10c91bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680) -> (continuation of Deferred@176229339 after retry RPC@75630850) -> (continuation of Deferred@1834273722 after retry RPC@2039088395) -> (continuation of Deferred@838945944 after retry RPC@453682574))) after lookup
18:30:29.748 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:29.749 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.749 [DEBUG - kudu-nio-5] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.749 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.867 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:29.868 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)[L]]}]
18:30:29.868 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=414), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [3ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [3ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [21ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [22ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [23ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [34ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [35ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [36ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [36ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [54ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [55ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [57ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [57ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [194ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [195ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [197ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [197ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [291ms] refreshing cache from master, [291ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [294ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [295ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [296ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [296ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [411ms] refreshing cache from master, [411ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [414ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1240697359(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@53f8b967 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680) -> (continuation of Deferred@176229339 after retry RPC@75630850) -> (continuation of Deferred@1834273722 after retry RPC@2039088395) -> (continuation of Deferred@838945944 after retry RPC@453682574) -> (continuation of Deferred@1389140906 after retry RPC@480535208), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10c91bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680) -> (continuation of Deferred@176229339 after retry RPC@75630850) -> (continuation of Deferred@1834273722 after retry RPC@2039088395) -> (continuation of Deferred@838945944 after retry RPC@453682574) -> (continuation of Deferred@1389140906 after retry RPC@480535208))) after lookup
18:30:29.869 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] connecting to peer
18:30:29.870 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.870 [DEBUG - kudu-nio-6] (Connection.java:709) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] cleaning up while in state CONNECTING due to: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:29.870 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285
18:30:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.630112  3356 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 9 FOLLOWER]: Starting pre-election (detected failure of leader bacacb109f8247dcb77a540bd9e605a1)
18:30:30.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.630617  3356 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.632511  3356 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:30.638018  3046 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111)
18:30:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.643900  3297 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" is_pre_election: true
18:30:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.644698  3297 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9df350dc58374330a22867bfa403ed8d in term 8.
18:30:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.645861  3045 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, a8fdfc4241d34c848f5eb3275d07e1f6; no voters: bacacb109f8247dcb77a540bd9e605a1
18:30:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.646459  3356 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 9 FOLLOWER]: Leader pre-election won for term 10
18:30:30.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.646754  3356 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 9 FOLLOWER]: Starting leader election (detected failure of leader bacacb109f8247dcb77a540bd9e605a1)
18:30:30.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.647094  3356 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 9 FOLLOWER]: Advancing to term 10
18:30:30.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.651564  3356 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:30.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.653111  3356 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 10 election: Requested vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65:45335)
18:30:30.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.654312  3297 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "9df350dc58374330a22867bfa403ed8d" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6"
18:30:30.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:30.654662  3046 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111) [suppressed 1 similar messages]
18:30:30.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.654881  3297 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 8 FOLLOWER]: Advancing to term 10
18:30:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:30.657493  3046 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111)
18:30:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.662307  3297 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9df350dc58374330a22867bfa403ed8d in term 10.
18:30:30.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.663219  3045 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9df350dc58374330a22867bfa403ed8d, a8fdfc4241d34c848f5eb3275d07e1f6; no voters: bacacb109f8247dcb77a540bd9e605a1
18:30:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.663894  3356 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 10 FOLLOWER]: Leader election won for term 10
18:30:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.664971  3356 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 10 LEADER]: Becoming Leader. State: Replica: 9df350dc58374330a22867bfa403ed8d, State: Running, Role: LEADER
18:30:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.665802  3356 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:30.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:30.675230  1287 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d reported cstate change: term changed from 9 to 10, leader changed from bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67) to 9df350dc58374330a22867bfa403ed8d (127.1.16.66). New cstate: current_term: 10 leader_uuid: "9df350dc58374330a22867bfa403ed8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: UNKNOWN } } }
18:30:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:31.185847  3046 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d -> Peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Couldn't send request to peer bacacb109f8247dcb77a540bd9e605a1. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:30:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:31.220906  3297 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 10 FOLLOWER]: Refusing update from remote peer 9df350dc58374330a22867bfa403ed8d: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
18:30:31.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:31.222224  3373 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
18:30:32.348 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:32.349 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:32.349 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2895), Traces: [0ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [3ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [3ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [21ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [22ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [23ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [34ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [35ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [36ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [36ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [54ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [55ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [57ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [57ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [194ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [195ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [197ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [197ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [291ms] refreshing cache from master, [291ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [294ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [295ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [296ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [296ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [411ms] refreshing cache from master, [411ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [414ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [416ms] sending RPC to server bacacb109f8247dcb77a540bd9e605a1, [417ms] received response from server bacacb109f8247dcb77a540bd9e605a1: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [417ms] delaying RPC due to: Network error: Failed to connect to peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285): Connection refused: /127.1.16.67:37285, [2891ms] refreshing cache from master, [2892ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [2895ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1240697359(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@53f8b967 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680) -> (continuation of Deferred@176229339 after retry RPC@75630850) -> (continuation of Deferred@1834273722 after retry RPC@2039088395) -> (continuation of Deferred@838945944 after retry RPC@453682574) -> (continuation of Deferred@1389140906 after retry RPC@480535208) -> (continuation of Deferred@227801960 after retry RPC@840792400), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10c91bd4 -> wakeup thread Time-limited test -> (continuation of Deferred@1484226065 after retry RPC@3725680) -> (continuation of Deferred@176229339 after retry RPC@75630850) -> (continuation of Deferred@1834273722 after retry RPC@2039088395) -> (continuation of Deferred@838945944 after retry RPC@453682574) -> (continuation of Deferred@1389140906 after retry RPC@480535208) -> (continuation of Deferred@227801960 after retry RPC@840792400))) after lookup
18:30:32.350 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:32.351 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] Successfully connected to peer
18:30:32.384 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x3729dc88, L:/127.0.0.1:46586 - R:/127.1.16.66:44881] using TOKEN/null
18:30:32.434 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.16.67:37285
18:30:32.435 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:32.436184  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/logs
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:30:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:30:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.67:37285
18:30:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.67
18:30:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37737
18:30:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.67
18:30:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:30:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:32.764109  3396 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:30:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:32.764734  3396 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:30:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:32.765365  3396 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:30:32.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:32.799026  3396 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:30:32.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:32.800042  3396 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.67
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:32.837604  3396 tablet_server_runner.cc:78] Tablet server non-default flags:
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.67:37285
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.67
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37737
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.67
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/logs
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:30:32.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:30:32.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:30:32.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:30:32.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:30:32.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:30:32.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:30:32.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:32.839314  3396 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:30:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:32.841015  3396 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:30:32.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:32.853084  3405 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:33.660583  3046 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d -> Peer bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285): Couldn't send request to peer bacacb109f8247dcb77a540bd9e605a1. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.67:37285: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
18:30:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:32.855360  3406 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:34.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:34.227252  3408 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:34.229291  3407 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1370 milliseconds
18:30:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.229427  3396 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:30:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.230530  3396 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:30:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.233709  3396 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:30:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.235051  3396 hybrid_clock.cc:648] HybridClock initialized: now 1783535434235021 us; error 57 us; skew 500 ppm
18:30:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.235795  3396 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:30:34.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.242740  3396 webserver.cc:534] Webserver started at http://127.1.16.67:37737/ using document root <none> and password file <none>
18:30:34.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.243567  3396 fs_manager.cc:362] Metadata directory not provided
18:30:34.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.243829  3396 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:30:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.251205  3396 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
18:30:34.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.255426  3418 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:30:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.256484  3396 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
18:30:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.256757  3396 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:30:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:47 on dist-test-slave-l44q"
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.258597  3396 fs_report.cc:389] FS layout report
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/wal
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/data
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:30:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:30:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.313657  3396 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:30:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.315074  3396 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:30:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.315491  3396 kserver.cc:163] Server-wide thread pool size limit: 3276
18:30:34.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.318617  3396 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:30:34.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.324582  3425 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:30:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.332862  3396 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:30:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.333176  3396 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.000s
18:30:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.333478  3396 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:30:34.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.338511  3396 ts_tablet_manager.cc:616] Registered 1 tablets
18:30:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.338775  3396 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
18:30:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.339157  3425 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Bootstrap starting.
18:30:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.424257  3425 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Log is configured to *not* fsync() on all Append() calls
18:30:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.515329  3396 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.67:37285
18:30:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.515494  3532 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.67:37285 every 8 connection(s)
18:30:34.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.530109  3396 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-2/data/info.pb
18:30:34.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.532419  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 3396
18:30:34.546 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:34.606 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:30:34.607 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.16.66:44881
18:30:34.607 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:30:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.608160  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 3019
18:30:34.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.630595  3425 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:30:34.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.630447  3533 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:30:34.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.631209  3533 heartbeater.cc:461] Registering TS with master...
18:30:34.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.631721  3425 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Bootstrap complete.
18:30:34.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.633076  3533 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:30:34.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.633570  3425 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Time spent bootstrapping tablet: real 0.295s	user 0.201s	sys 0.035s
18:30:34.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.650647  3536 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:30:34.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.651055  3536 heartbeater.cc:461] Registering TS with master...
18:30:34.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.652407  3536 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:30:34.658 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] handling channelInactive
18:30:34.659 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state READY due to: connection closed
18:30:34.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.655939  1288 ts_manager.cc:194] Re-registered known tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:30:34.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.656685  1209 ts_manager.cc:194] Re-registered known tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:30:34.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.657299  3539 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:30:34.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.657665  3539 heartbeater.cc:461] Registering TS with master...
18:30:34.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.658638  3539 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:30:34.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.663497  1288 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.67:46321
18:30:34.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.663620  1133 ts_manager.cc:194] Re-registered known tserver with Master: bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285)
18:30:34.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:34.668 [DEBUG - Time-limited test] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.665813  3425 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.668733  3425 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: bacacb109f8247dcb77a540bd9e605a1, State: Initialized, Role: FOLLOWER
18:30:34.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.669 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.669677  3425 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.672262  3533 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:30:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:34.672873  3425 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1: Time spent starting tablet: real 0.039s	user 0.021s	sys 0.005s
18:30:34.687 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:34.688 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:34.688 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [1ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1416113915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@131bc596 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11c5e8fb -> wakeup thread Time-limited test)) after lookup
18:30:34.688 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:34.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.690 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.708 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:34.709 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:34.709 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [1ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [20ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [22ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [22ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1416113915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@131bc596 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1669777418 after retry RPC@1572497073), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11c5e8fb -> wakeup thread Time-limited test -> (continuation of Deferred@1669777418 after retry RPC@1572497073))) after lookup
18:30:34.710 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:34.711 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.711 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.711 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.727 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:34.727 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:34.728 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=60), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [1ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [20ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [22ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [22ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [41ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [43ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [43ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [59ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1416113915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@131bc596 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1669777418 after retry RPC@1572497073) -> (continuation of Deferred@1251124568 after retry RPC@1150541141), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11c5e8fb -> wakeup thread Time-limited test -> (continuation of Deferred@1669777418 after retry RPC@1572497073) -> (continuation of Deferred@1251124568 after retry RPC@1150541141))) after lookup
18:30:34.728 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:34.729 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.729 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.730 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.848 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:34.848 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:34.849 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=180), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [1ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [20ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [22ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [22ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [41ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [43ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [43ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [59ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [60ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [62ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [62ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [176ms] refreshing cache from master, [176ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [180ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1416113915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@131bc596 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1669777418 after retry RPC@1572497073) -> (continuation of Deferred@1251124568 after retry RPC@1150541141) -> (continuation of Deferred@739108064 after retry RPC@1991179585), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11c5e8fb -> wakeup thread Time-limited test -> (continuation of Deferred@1669777418 after retry RPC@1572497073) -> (continuation of Deferred@1251124568 after retry RPC@1150541141) -> (continuation of Deferred@739108064 after retry RPC@1991179585))) after lookup
18:30:34.849 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:34.850 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.850 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:34.850 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:35.088 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:35.088 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)[L],a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335),bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:35.089 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=421), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [1ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [20ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [22ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [22ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [41ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [43ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [43ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [59ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [60ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [62ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [62ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [176ms] refreshing cache from master, [176ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [180ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [181ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [182ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [182ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [416ms] refreshing cache from master, [416ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [420ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1416113915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@131bc596 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1669777418 after retry RPC@1572497073) -> (continuation of Deferred@1251124568 after retry RPC@1150541141) -> (continuation of Deferred@739108064 after retry RPC@1991179585) -> (continuation of Deferred@640124647 after retry RPC@660747159), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11c5e8fb -> wakeup thread Time-limited test -> (continuation of Deferred@1669777418 after retry RPC@1572497073) -> (continuation of Deferred@1251124568 after retry RPC@1150541141) -> (continuation of Deferred@739108064 after retry RPC@1991179585) -> (continuation of Deferred@640124647 after retry RPC@660747159))) after lookup
18:30:35.089 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] connecting to peer
18:30:35.090 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:35.090 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:35.090 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881) for tablet 41dcfd2a13434555b4e5a2b746a3745f: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881
18:30:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:35.563756  1240 debug-util.cc:398] Leaking SignalData structure 0x7b0800097980 after lost signal to thread 1175
18:30:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:35.564711  1240 debug-util.cc:398] Leaking SignalData structure 0x7b0800098340 after lost signal to thread 1243
18:30:35.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.739880  3546 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 9df350dc58374330a22867bfa403ed8d)
18:30:35.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.740393  3546 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:35.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.742654  3546 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:35.747722  3231 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:30:35.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:35.751533  3231 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:30:35.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.755280  3488 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1" is_pre_election: true
18:30:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.755939  3488 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a8fdfc4241d34c848f5eb3275d07e1f6 in term 9.
18:30:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.757160  3231 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a8fdfc4241d34c848f5eb3275d07e1f6, bacacb109f8247dcb77a540bd9e605a1; no voters: 9df350dc58374330a22867bfa403ed8d
18:30:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.757833  3546 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 10 FOLLOWER]: Leader pre-election won for term 11
18:30:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.758148  3546 raft_consensus.cc:493] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 9df350dc58374330a22867bfa403ed8d)
18:30:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.758526  3546 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 10 FOLLOWER]: Advancing to term 11
18:30:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.762995  3546 raft_consensus.cc:515] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.764526  3546 leader_election.cc:290] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 11 election: Requested vote from peers bacacb109f8247dcb77a540bd9e605a1 (127.1.16.67:37285), 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.765316  3488 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "41dcfd2a13434555b4e5a2b746a3745f" candidate_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "bacacb109f8247dcb77a540bd9e605a1"
18:30:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.765725  3488 raft_consensus.cc:3060] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 9 FOLLOWER]: Advancing to term 11
18:30:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:35.768599  3231 leader_election.cc:336] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111)
18:30:35.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.771812  3488 raft_consensus.cc:2468] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a8fdfc4241d34c848f5eb3275d07e1f6 in term 11.
18:30:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.772842  3231 leader_election.cc:304] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a8fdfc4241d34c848f5eb3275d07e1f6, bacacb109f8247dcb77a540bd9e605a1; no voters: 9df350dc58374330a22867bfa403ed8d
18:30:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.773772  3546 raft_consensus.cc:2804] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 11 FOLLOWER]: Leader election won for term 11
18:30:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.774708  3546 raft_consensus.cc:697] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [term 11 LEADER]: Becoming Leader. State: Replica: a8fdfc4241d34c848f5eb3275d07e1f6, State: Running, Role: LEADER
18:30:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.775547  3546 consensus_queue.cc:237] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:35.784444  1288 catalog_manager.cc:5696] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 reported cstate change: term changed from 10 to 11, leader changed from 9df350dc58374330a22867bfa403ed8d (127.1.16.66) to a8fdfc4241d34c848f5eb3275d07e1f6 (127.1.16.65). New cstate: current_term: 11 leader_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } health_report { overall_health: HEALTHY } } }
18:30:36.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 41dcfd2a13434555b4e5a2b746a3745f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770' with partition [<start>, <end>)
18:30:36.048 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=41dcfd2a13434555b4e5a2b746a3745f@[9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881),a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)[L],bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)]}]
18:30:36.049 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=41dcfd2a13434555b4e5a2b746a3745f, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1380), Traces: [0ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [1ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [19ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [20ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [22ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [22ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [41ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [43ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [43ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [59ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [60ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [62ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [62ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [176ms] refreshing cache from master, [176ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [180ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [181ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [182ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [182ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [416ms] refreshing cache from master, [416ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [420ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, [421ms] sending RPC to server 9df350dc58374330a22867bfa403ed8d, [422ms] received response from server 9df350dc58374330a22867bfa403ed8d: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [422ms] delaying RPC due to: Network error: Failed to connect to peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881): Connection refused: /127.1.16.66:44881, [1376ms] refreshing cache from master, [1376ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.16.124:45813, [1380ms] Sub RPC GetTableLocations: received response from server master-127.1.16.124:45813: OK, deferred=Deferred@1416113915(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f42a036aeb8@131bc596 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1669777418 after retry RPC@1572497073) -> (continuation of Deferred@1251124568 after retry RPC@1150541141) -> (continuation of Deferred@739108064 after retry RPC@1991179585) -> (continuation of Deferred@640124647 after retry RPC@660747159) -> (continuation of Deferred@819152341 after retry RPC@102446803), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11c5e8fb -> wakeup thread Time-limited test -> (continuation of Deferred@1669777418 after retry RPC@1572497073) -> (continuation of Deferred@1251124568 after retry RPC@1150541141) -> (continuation of Deferred@739108064 after retry RPC@1991179585) -> (continuation of Deferred@640124647 after retry RPC@660747159) -> (continuation of Deferred@819152341 after retry RPC@102446803))) after lookup
18:30:36.049 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] connecting to peer
18:30:36.050 [DEBUG - kudu-nio-6] (Connection.java:757) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] Successfully connected to peer
18:30:36.077 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x69a63395, L:/127.0.0.1:41782 - R:/127.1.16.65:45335] using TOKEN/null
18:30:36.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:36.087966  3488 raft_consensus.cc:1275] T 41dcfd2a13434555b4e5a2b746a3745f P bacacb109f8247dcb77a540bd9e605a1 [term 11 FOLLOWER]: Refusing update from remote peer a8fdfc4241d34c848f5eb3275d07e1f6: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 42. (index mismatch)
18:30:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:36.089865  3551 consensus_queue.cc:1048] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
18:30:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:36.090435  3231 consensus_peers.cc:597] T 41dcfd2a13434555b4e5a2b746a3745f P a8fdfc4241d34c848f5eb3275d07e1f6 -> Peer 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881): Couldn't send request to peer 9df350dc58374330a22867bfa403ed8d. Status: Network error: Client connection negotiation failed: client connection to 127.1.16.66:44881: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:30:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:36.106721  3566 mvcc.cc:204] Tried to move back new op lower bound from 7305361131069878272 to 7305361124014694400. Current Snapshot: MvccSnapshot[applied={T|T < 7305361131069878272}]
18:30:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:36.112438  3564 mvcc.cc:204] Tried to move back new op lower bound from 7305361146197614592 to 7305361144944857088. Current Snapshot: MvccSnapshot[applied={T|T < 7305361146197614592}]
18:30:36.154 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.16.66:44881
18:30:36.154 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:36.155821  1089 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/logs
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.66:44881
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.66
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36159
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.66
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:30:36.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:36.469173  3572 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:30:36.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:36.469650  3572 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:30:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:36.470479  3572 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:30:36.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:36.503971  3572 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:30:36.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:36.504995  3572 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.16.66
18:30:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:36.542342  3572 tablet_server_runner.cc:78] Tablet server non-default flags:
18:30:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:30:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.16.84:37805
18:30:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.16.66:44881
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.16.66
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36159
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.16.126:39121,127.1.16.125:33673,127.1.16.124:45813
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.16.66
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/logs
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:05:40 UTC on e7f111948823
18:30:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16871
18:30:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:30:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:36.543684  3572 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:30:36.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:36.545552  3572 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:30:36.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:36.559422  3578 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:36.560189  3579 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:37.964996  3581 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:30:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:37.970109  3572 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.410s	user 0.459s	sys 0.925s
18:30:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:37.970472  3572 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.410s	user 0.459s	sys 0.925s
18:30:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:30:37.970435  3580 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1409 milliseconds
18:30:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:37.970819  3572 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:30:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:37.972502  3572 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:30:37.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:37.975267  3572 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:30:37.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:37.976722  3572 hybrid_clock.cc:648] HybridClock initialized: now 1783535437976674 us; error 35 us; skew 500 ppm
18:30:37.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:37.977874  3572 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:30:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:37.986958  3572 webserver.cc:534] Webserver started at http://127.1.16.66:36159/ using document root <none> and password file <none>
18:30:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:37.988274  3572 fs_manager.cc:362] Metadata directory not provided
18:30:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:37.988600  3572 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:30:38.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.000228  3572 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.006s	sys 0.000s
18:30:38.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.006193  3592 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:30:38.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.007570  3572 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
18:30:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.008054  3572 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data,/tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9df350dc58374330a22867bfa403ed8d"
18:30:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:29:45 on dist-test-slave-l44q"
18:30:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.011188  3572 fs_report.cc:389] FS layout report
18:30:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:30:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/wal
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/data
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:30:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:30:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.091432  3572 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:30:38.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.093523  3572 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:30:38.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.094121  3572 kserver.cc:163] Server-wide thread pool size limit: 3276
18:30:38.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.097744  3572 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:30:38.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.105669  3599 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:30:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.114029  3572 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:30:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.114313  3572 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.000s	sys 0.002s
18:30:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.114645  3572 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:30:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.119040  3572 ts_tablet_manager.cc:616] Registered 1 tablets
18:30:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.119208  3572 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
18:30:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.119656  3599 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap starting.
18:30:38.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.210304  3599 log.cc:826] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Log is configured to *not* fsync() on all Append() calls
18:30:38.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.294574  3572 rpc_server.cc:307] RPC server started. Bound to: 127.1.16.66:44881
18:30:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.294698  3706 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.16.66:44881 every 8 connection(s)
18:30:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.298228  3572 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5ZdiiQ/test-tmp/mini-kudu-cluster7897566168684182020/ts-1/data/info.pb
18:30:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.307082  1089 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu as pid 3572
18:30:38.308 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:30:38.348 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:30:38.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.367462  3708 heartbeater.cc:344] Connected to a master server at 127.1.16.125:33673
18:30:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.368216  3708 heartbeater.cc:461] Registering TS with master...
18:30:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.370170  3708 heartbeater.cc:507] Master 127.1.16.125:33673 requested a full tablet report, sending...
18:30:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.375357  1208 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.375427  3707 heartbeater.cc:344] Connected to a master server at 127.1.16.124:45813
18:30:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.375766  3707 heartbeater.cc:461] Registering TS with master...
18:30:38.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.376852  3707 heartbeater.cc:507] Master 127.1.16.124:45813 requested a full tablet report, sending...
18:30:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.380187  1288 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:38.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.383595  1288 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.16.66:55191
18:30:38.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.384668  3709 heartbeater.cc:344] Connected to a master server at 127.1.16.126:39121
18:30:38.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.385033  3709 heartbeater.cc:461] Registering TS with master...
18:30:38.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.386020  3709 heartbeater.cc:507] Master 127.1.16.126:39121 requested a full tablet report, sending...
18:30:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.391992  1134 ts_manager.cc:194] Re-registered known tserver with Master: 9df350dc58374330a22867bfa403ed8d (127.1.16.66:44881)
18:30:38.398 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783535388770, tablet=41dcfd2a13434555b4e5a2b746a3745f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:30:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.403343  3599 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:30:38.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.404693  3599 tablet_bootstrap.cc:492] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Bootstrap complete.
18:30:38.406 [DEBUG - main] (Connection.java:575) [peer master-127.1.16.124:45813(127.1.16.124:45813)] disconnecting while in state READY
18:30:38.407 [DEBUG - main] (Connection.java:575) [peer 9df350dc58374330a22867bfa403ed8d(127.1.16.66:44881)] disconnecting while in state TERMINATED
18:30:38.408 [DEBUG - main] (Connection.java:575) [peer master-127.1.16.125:33673(127.1.16.125:33673)] disconnecting while in state READY
18:30:38.408 [DEBUG - main] (Connection.java:575) [peer bacacb109f8247dcb77a540bd9e605a1(127.1.16.67:37285)] disconnecting while in state TERMINATED
18:30:38.408 [DEBUG - main] (Connection.java:575) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] disconnecting while in state READY
18:30:38.408 [DEBUG - main] (Connection.java:575) [peer master-127.1.16.126:39121(127.1.16.126:39121)] disconnecting while in state READY
18:30:38.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.406445  3599 ts_tablet_manager.cc:1403] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Time spent bootstrapping tablet: real 0.287s	user 0.181s	sys 0.059s
18:30:38.410 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.16.124:45813(127.1.16.124:45813)] handling channelInactive
18:30:38.410 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.1.16.125:33673(127.1.16.125:33673)] handling channelInactive
18:30:38.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.16.124:45813(127.1.16.124:45813)] cleaning up while in state READY due to: connection closed
18:30:38.410 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.1.16.125:33673(127.1.16.125:33673)] cleaning up while in state READY due to: connection closed
18:30:38.411 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.16.126:39121(127.1.16.126:39121)] handling channelInactive
18:30:38.412 [DEBUG - kudu-nio-6] (Connection.java:239) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] handling channelInactive
18:30:38.412 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a8fdfc4241d34c848f5eb3275d07e1f6(127.1.16.65:45335)] cleaning up while in state READY due to: connection closed
18:30:38.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.16.126:39121(127.1.16.126:39121)] cleaning up while in state READY due to: connection closed
18:30:38.413 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
18:30:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.416013  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 3208
18:30:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.433211  3599 raft_consensus.cc:359] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.436651  3599 raft_consensus.cc:740] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9df350dc58374330a22867bfa403ed8d, State: Initialized, Role: FOLLOWER
18:30:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.437608  3599 consensus_queue.cc:260] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bacacb109f8247dcb77a540bd9e605a1" member_type: VOTER last_known_addr { host: "127.1.16.67" port: 37285 } } peers { permanent_uuid: "9df350dc58374330a22867bfa403ed8d" member_type: VOTER last_known_addr { host: "127.1.16.66" port: 44881 } } peers { permanent_uuid: "a8fdfc4241d34c848f5eb3275d07e1f6" member_type: VOTER last_known_addr { host: "127.1.16.65" port: 45335 } }
18:30:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.440799  3707 heartbeater.cc:499] Master 127.1.16.124:45813 was elected leader, sending a full tablet report...
18:30:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.441373  3599 ts_tablet_manager.cc:1434] T 41dcfd2a13434555b4e5a2b746a3745f P 9df350dc58374330a22867bfa403ed8d: Time spent starting tablet: real 0.035s	user 0.029s	sys 0.001s
18:30:38.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.455883  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 3572
18:30:38.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.481761  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 3396
18:30:38.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.512966  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 1101
18:30:38.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.544629  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 1171
18:30:38.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:30:38.577813  1089 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5ZdiiQ/build/tsan/bin/kudu with pid 1246
18:30:39.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T18:30:39Z chronyd exiting

Time: 69.394

OK (1 test)

