Test failure: j11_dtests_large.pending_range_test.TestPendingRangeMovements

ccmlib.node.TimeoutError: 23 Nov 2023 21:15:48 [node1] after 10.01/10 seconds Missing: ['Moving .* to -634023222112864484'] not found in system.log:
Head: INFO [RMI TCP Connection(2)-127.0.0.1] 2023-11-23
Tail: ...1-23 21:15:39,483 BigFormat.java:324 - Deleting sstable: /tmp/dtest-ovho0emz/test/node1/data0/system/local-7ad54392bcdd35a684174e047860b377/oa-11-big
self = <pending_range_test.TestPendingRangeMovements object at 0x7f0f2c335d50>

@pytest.mark.resource_intensive
def test_pending_range(self):
"""
@jira_ticket CASSANDRA-10887
"""
cluster = self.cluster
 # If we are on 2.1, we need to set the log level to debug or higher, as debug.log does not exist.
if cluster.version() < '2.2':
cluster.set_log_level('DEBUG')

 # Create 5 node cluster
ring_delay_ms = 3_600_000 # 1 hour
cluster.populate(5).start(jvm_args=['-Dcassandra.ring_delay_ms={}'.format(ring_delay_ms)])
node1, node2 = cluster.nodelist()[0:2]

 # Set up RF=3 keyspace
session = self.patient_cql_connection(node1)
create_ks(session, 'ks', 3)

session.execute("CREATE TABLE users (login text PRIMARY KEY, email text, name text, login_count int)")
 # We use the partition key 'jdoe3' because it belongs to node1.
 # The key MUST belong to node1 to repro the bug.
session.execute("INSERT INTO users (login, email, name, login_count) VALUES ('jdoe3', 'jdoe@abc.com', 'Jane Doe', 1) IF NOT EXISTS;")

lwt_query = SimpleStatement("UPDATE users SET email = 'janedoe@abc.com' WHERE login = 'jdoe3' IF email = 'jdoe@abc.com'")
 # Show we can execute LWT no problem
for i in range(1000):
session.execute(lwt_query)

token = '-634023222112864484'

mark = node1.mark_log()
 # Move a node without waiting for the response of nodetool, so we don't have to wait for ring_delay
threading.Thread(target=(lambda: node1.nodetool('move {}'.format(token)))).start()

 # Watch the log so we know when the node is moving
> node1.watch_log_for('Moving .* to {}'.format(token), timeout=10, from_mark=mark)

pending_range_test.py:55: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../env3.7/lib/python3.7/site-packages/ccmlib/node.py:605: in watch_log_for
head=reads[:50], tail="..."+reads[len(reads)-150:]))
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

start = 1700774138.162831, timeout = 10
msg = "Missing: ['Moving .* to -634023222112864484'] not found in system.log:\n Head: INFO [RMI TCP Connection(2)-127.0.0.1...324 - Deleting sstable: /tmp/dtest-ovho0emz/test/node1/data0/system/local-7ad54392bcdd35a684174e047860b377/oa-11-big\n"
node = 'node1'

@staticmethod
def raise_if_passed(start, timeout, msg, node=None):
if start + timeout < time.time():
> raise TimeoutError.create(start, timeout, msg, node)
E ccmlib.node.TimeoutError: 23 Nov 2023 21:15:48 [node1] after 10.01/10 seconds Missing: ['Moving .* to -634023222112864484'] not found in system.log:
E Head: INFO [RMI TCP Connection(2)-127.0.0.1] 2023-11-23
E Tail: ...1-23 21:15:39,483 BigFormat.java:324 - Deleting sstable: /tmp/dtest-ovho0emz/test/node1/data0/system/local-7ad54392bcdd35a684174e047860b377/oa-11-big

../env3.7/lib/python3.7/site-packages/ccmlib/node.py:56: TimeoutError