test: add tests for migration FSM paths

Signed-off-by: William Henderson <william.henderson@nutanix.com>

test/py/libvfio_user.py

@@ -197,11 +197,22 @@ def is_32bit():
 
 VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP = (1 << 0)
 
+# enum vfio_user_device_mig_state
+VFIO_USER_DEVICE_STATE_ERROR = 0
+VFIO_USER_DEVICE_STATE_STOP = 1
+VFIO_USER_DEVICE_STATE_RUNNING = 2
+VFIO_USER_DEVICE_STATE_STOP_COPY = 3
+VFIO_USER_DEVICE_STATE_RESUMING = 4
+VFIO_USER_DEVICE_STATE_RUNNING_P2P = 5
+VFIO_USER_DEVICE_STATE_PRE_COPY = 6
+VFIO_USER_DEVICE_STATE_PRE_COPY_P2P = 7
+
 VFIO_DEVICE_FEATURE_MASK = 0xffff
 VFIO_DEVICE_FEATURE_GET = (1 << 16)
 VFIO_DEVICE_FEATURE_SET = (1 << 17)
 VFIO_DEVICE_FEATURE_PROBE = (1 << 18)
 
+VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE = 2
 VFIO_DEVICE_FEATURE_DMA_LOGGING_START = 6
 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP = 7
 VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT = 8
@@ -210,6 +221,13 @@ def is_32bit():
 VFIO_USER_IO_FD_TYPE_IOREGIONFD = 1
 VFIO_USER_IO_FD_TYPE_IOEVENTFD_SHADOW = 2
 
+# enum vfu_migr_state_t
+VFU_MIGR_STATE_STOP = 0
+VFU_MIGR_STATE_RUNNING = 1
+VFU_MIGR_STATE_STOP_AND_COPY = 2
+VFU_MIGR_STATE_PRE_COPY = 3
+VFU_MIGR_STATE_RESUME = 4
+
 
 # enum vfu_dev_irq_type
 VFU_DEV_INTX_IRQ = 0
@@ -556,6 +574,14 @@ class vfio_user_device_feature(Structure):
  ]
 
 
+class vfio_user_device_feature_mig_state(Structure):
+ _pack_ = 1
+ _fields_ = [
+ ("device_state", c.c_uint32),
+ ("data_fd", c.c_uint32),
+ ]
+
+
 class vfio_user_device_feature_dma_logging_control(Structure):
  _pack_ = 1
  _fields_ = [

test/py/meson.build

@@ -37,6 +37,7 @@ python_tests = [
  'test_dma_map.py',
  'test_dma_unmap.py',
  'test_irq_trigger.py',
+ 'test_migration.py',
  'test_negotiate.py',
  'test_pci_caps.py',
  'test_pci_ext_caps.py',

test/py/test_migration.py

@@ -0,0 +1,244 @@
+#
+# Copyright (c) 2021 Nutanix Inc. All rights reserved.
+#
+# Authors: John Levon <john.levon@nutanix.com>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Nutanix nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+#
+
+from libvfio_user import *
+from collections import deque
+
+ctx = None
+sock = None
+current_state = None
+path = []
+
+
+UNREACHABLE_STATES = {
+ VFIO_USER_DEVICE_STATE_ERROR,
+ VFIO_USER_DEVICE_STATE_PRE_COPY_P2P,
+ VFIO_USER_DEVICE_STATE_RUNNING_P2P
+}
+
+
+@transition_cb_t
+def migr_trans_cb(_ctx, state):
+ global current_state, path
+
+ if state == VFU_MIGR_STATE_STOP:
+ state = VFIO_USER_DEVICE_STATE_STOP
+ elif state == VFU_MIGR_STATE_RUNNING:
+ state = VFIO_USER_DEVICE_STATE_RUNNING
+ elif state == VFU_MIGR_STATE_STOP_AND_COPY:
+ state = VFIO_USER_DEVICE_STATE_STOP_COPY
+ elif state == VFU_MIGR_STATE_RESUME:
+ state = VFIO_USER_DEVICE_STATE_RESUMING
+ elif state == VFU_MIGR_STATE_PRE_COPY:
+ state = VFIO_USER_DEVICE_STATE_PRE_COPY
+ else:
+ assert False
+
+ current_state = state
+
+ path.append(state)
+
+ return 0
+
+
+@read_data_cb_t
+def migr_read_data_cb(_ctx, _buf, _count, _offset):
+ return
+
+
+@write_data_cb_t
+def migr_write_data_cb(_ctx, _buf, _count, _offset):
+ return
+
+
+def test_migration_setup():
+ global ctx, sock
+
+ ctx = vfu_create_ctx(flags=LIBVFIO_USER_FLAG_ATTACH_NB)
+ assert ctx is not None
+
+ cbs = vfu_migration_callbacks_t()
+ cbs.version = VFU_MIGR_CALLBACKS_VERS
+ cbs.transition = migr_trans_cb
+ cbs.read_data = migr_read_data_cb
+ cbs.write_data = migr_write_data_cb
+
+ ret = vfu_setup_device_migration_callbacks(ctx, 0, cbs)
+ assert ret == 0
+
+ vfu_setup_device_quiesce_cb(ctx)
+
+ ret = vfu_realize_ctx(ctx)
+ assert ret == 0
+
+ sock = connect_client(ctx)
+
+
+def get_server_shortest_path(a, b, expectA=0, expectB=0):
+ global ctx, sock, path
+
+ feature = vfio_user_device_feature(
+ argsz=len(vfio_user_device_feature()) +
+ len(vfio_user_device_feature_mig_state()),
+ flags=VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE
+ )
+
+ if current_state == VFIO_USER_DEVICE_STATE_STOP_COPY and \
+ a == VFIO_USER_DEVICE_STATE_PRE_COPY:
+ # The transition STOP_COPY -> PRE_COPY is explicitly blocked so we
+ # advance one state to get around this in order to set up the test.
+ payload = vfio_user_device_feature_mig_state(
+ device_state=VFIO_USER_DEVICE_STATE_STOP
+ )
+ msg(ctx, sock, VFIO_USER_DEVICE_FEATURE,
+ bytes(feature) + bytes(payload))
+
+ payload = vfio_user_device_feature_mig_state(device_state=a)
+ msg(ctx, sock, VFIO_USER_DEVICE_FEATURE, bytes(feature) + bytes(payload),
+ expect=expectA)
+
+ if expectA != 0:
+ return None
+
+ path = []
+
+ payload = vfio_user_device_feature_mig_state(device_state=b)
+ msg(ctx, sock, VFIO_USER_DEVICE_FEATURE, bytes(feature) + bytes(payload),
+ expect=expectB)
+
+ return path.copy()
+
+
+def test_migration_shortest_state_transition_paths():
+ """
+ The spec dictates that complex state transitions are to be implemented as
+ combinations of the defined direct transitions, with the path selected
+ according to the following rules:
+
+ - Select the shortest path.
+ - The path cannot have saving group states as interior arcs, only start/end
+ states.
+
+ This test implements a breadth-first search to ensure that the paths taken
+ by the implementation correctly follow these rules.
+ """
+
+ global ctx, sock
+
+ # states (vertices)
+ V = {VFIO_USER_DEVICE_STATE_ERROR, VFIO_USER_DEVICE_STATE_STOP,
+ VFIO_USER_DEVICE_STATE_RUNNING, VFIO_USER_DEVICE_STATE_STOP_COPY,
+ VFIO_USER_DEVICE_STATE_RESUMING, VFIO_USER_DEVICE_STATE_RUNNING_P2P,
+ VFIO_USER_DEVICE_STATE_PRE_COPY, VFIO_USER_DEVICE_STATE_PRE_COPY_P2P}
+
+ # allowed direct transitions (edges)
+ E = {
+ VFIO_USER_DEVICE_STATE_ERROR: set(),
+ VFIO_USER_DEVICE_STATE_STOP: {
+ VFIO_USER_DEVICE_STATE_RUNNING,
+ VFIO_USER_DEVICE_STATE_STOP_COPY,
+ VFIO_USER_DEVICE_STATE_RESUMING
+ },
+ VFIO_USER_DEVICE_STATE_RUNNING: {
+ VFIO_USER_DEVICE_STATE_STOP,
+ VFIO_USER_DEVICE_STATE_PRE_COPY
+ },
+ VFIO_USER_DEVICE_STATE_STOP_COPY: {VFIO_USER_DEVICE_STATE_STOP},
+ VFIO_USER_DEVICE_STATE_RESUMING: {VFIO_USER_DEVICE_STATE_STOP},
+ VFIO_USER_DEVICE_STATE_RUNNING_P2P: set(),
+ VFIO_USER_DEVICE_STATE_PRE_COPY: {
+ VFIO_USER_DEVICE_STATE_RUNNING,
+ VFIO_USER_DEVICE_STATE_STOP_COPY
+ },
+ VFIO_USER_DEVICE_STATE_PRE_COPY_P2P: set()
+ }
+
+ # "saving states" which cannot be internal arcs
+ S = {VFIO_USER_DEVICE_STATE_PRE_COPY, VFIO_USER_DEVICE_STATE_STOP_COPY}
+
+ for source in V:
+ back = {v: None for v in V}
+ queue = deque([(source, None)])
+
+ while len(queue) > 0:
+ (curr, prev) = queue.popleft()
+ back[curr] = prev
+ for nxt in E[curr]:
+ if back[nxt] is None and (curr == source or curr not in S):
+ queue.append((nxt, curr))
+
+ for target in V:
+ if source == VFIO_USER_DEVICE_STATE_STOP_COPY \
+ and target == VFIO_USER_DEVICE_STATE_PRE_COPY:
+ # test for this transition being blocked in a separate test
+ continue
+
+ if back[target] is not None:
+ seq = deque([])
+ curr = target
+ while curr != source:
+ seq.appendleft(curr)
+ curr = back[curr]
+
+ server_seq = get_server_shortest_path(source, target)
+
+ assert len(seq) == len(server_seq)
+ assert all(seq[i] == server_seq[i] for i in range(len(seq)))
+ else:
+ expectA = 22 if source in UNREACHABLE_STATES else 0
+
+ get_server_shortest_path(source, target, expectA=expectA,
+ expectB=22)
+
+
+def test_migration_stop_copy_to_pre_copy_blocked():
+ global ctx, sock
+
+ feature = vfio_user_device_feature(
+ argsz=len(vfio_user_device_feature()) +
+ len(vfio_user_device_feature_mig_state()),
+ flags=VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE
+ )
+ payload = vfio_user_device_feature_mig_state(
+ device_state=VFIO_USER_DEVICE_STATE_STOP_COPY
+ )
+ msg(ctx, sock, VFIO_USER_DEVICE_FEATURE, bytes(feature) + bytes(payload))
+
+ payload = vfio_user_device_feature_mig_state(
+ device_state=VFIO_USER_DEVICE_STATE_PRE_COPY
+ )
+ msg(ctx, sock, VFIO_USER_DEVICE_FEATURE, bytes(feature) + bytes(payload),
+ expect=22)
+
+
+def test_migration_cleanup():
+ disconnect_client(ctx, sock)
+ vfu_destroy_ctx(ctx)
+
+# ex: set tabstop=4 shiftwidth=4 softtabstop=4 expandtab:

test/unit-tests.c

@@ -537,17 +537,6 @@ test_setup_migration_callbacks(void **state)
  assert_int_equal(p->v->migration->state, VFIO_USER_DEVICE_STATE_RUNNING);
 }
 
-static void
-test_setup_migration_callbacks_resuming(void **state)
-{
- struct test_setup_migr_reg_dat *p = *state;
- int r = vfu_setup_device_migration_callbacks(p->v,
- LIBVFIO_USER_MIG_FLAG_START_RESUMING, &p->c);
- assert_int_equal(0, r);
- assert_non_null(p->v->migration);
- assert_int_equal(p->v->migration->state, VFIO_USER_DEVICE_STATE_RESUMING);
-}
-
 static void
 test_handle_device_state(void **state)
 {
@@ -631,7 +620,7 @@ test_handle_mig_data_read_too_long(void **state) {
 
  migr->state = VFIO_USER_DEVICE_STATE_PRE_COPY;
  r = handle_mig_data_read(p->v, m);
- assert_int_equal(-EINVAL, r);
+ assert_int_equal(-1, r);
 }
 
 static void
@@ -654,11 +643,11 @@ test_handle_mig_data_read_invalid_state(void **state) {
 
  migr->state = VFIO_USER_DEVICE_STATE_RUNNING;
  r = handle_mig_data_read(p->v, m);
- assert_int_equal(-EINVAL, r);
+ assert_int_equal(-1, r);
 
  migr->state = VFIO_USER_DEVICE_STATE_STOP;
  r = handle_mig_data_read(p->v, m);
- assert_int_equal(-EINVAL, r);
+ assert_int_equal(-1, r);
 }
 
 static void
@@ -719,7 +708,7 @@ test_handle_mig_data_write_invalid_state(void **state)
 
  migr->state = VFIO_USER_DEVICE_STATE_RUNNING;
  r = handle_mig_data_write(p->v, m);
- assert_int_equal(-EINVAL, r);
+ assert_int_equal(-1, r);
 }
 
 static void
@@ -827,9 +816,6 @@ main(void)
  cmocka_unit_test_setup_teardown(test_setup_migration_callbacks,
  setup_test_setup_migration,
  teardown_test_setup_migration),
- cmocka_unit_test_setup_teardown(test_setup_migration_callbacks_resuming,
- setup_test_setup_migration,
- teardown_test_setup_migration),
  cmocka_unit_test_setup_teardown(test_handle_device_state,
  setup_test_setup_migration,
  teardown_test_setup_migration),