feat+refactor: do not assert source VM not starting up, migrator issues ERROR on that (retried)

ci/lib.py

@@ -226,7 +226,7 @@ def wait_for_ostack_server_status(ostack_connection, server_name_or_id, server_s
         int_server_status = ostack_connection.compute.find_server(int_server.id).status
         if int_server_status == server_status:
             break
-        time.sleep(5)
+        time.sleep(10)
 
     return int_server_status
 
@@ -241,7 +241,7 @@ def wait_for_ostack_volume_status(ostack_connection, volume_name_or_id, volume_s
         int_volume_status = ostack_connection.block_storage.find_volume(int_volume.id).status
         if int_volume_status == volume_status:
             break
-        time.sleep(5)
+        time.sleep(10)
 
 
     return int_volume_status

ci/project-migrator.py

@@ -191,8 +191,8 @@ def main(args):
         # source VM stop, wait for SHUTOFF
         if i_source_server_detail.status != 'SHUTOFF':
             source_project_conn.compute.stop_server(i_source_server_detail)
-            args.logger.info(f"F.33 Source OpenStack VM server has been stopped")
-            lib.log_or_assert(args, "F.33 Source OpenStack VM server stopped (reached SHUTOFF state)",
+            args.logger.info(f"F.33 Source OpenStack VM server (name:{i_source_server_detail.name}) requested to stop")
+            lib.log_or_assert(args, f"F.33 Source OpenStack VM server (name:{i_source_server_detail.name}) stopped (reached SHUTOFF state)",
                               lib.wait_for_ostack_server_status(source_project_conn, i_source_server.id, 'SHUTOFF') == "SHUTOFF")
 
         # volume migration (browse i_server_block_device_mappings)
@@ -200,19 +200,13 @@ def main(args):
             clib.migrate_rbd_image(args, i_server_block_device_mapping)
 
         # start server in source cloud, wait for back being 'ACTIVE'
+        i_src_server_state_active_check = False
         if i_source_server_detail.status != source_project_conn.compute.find_server(i_source_server.id).status and \
            not args.source_servers_left_shutoff:
             if i_source_server_detail.status == 'ACTIVE':
                 source_project_conn.compute.start_server(i_source_server_detail)
-                args.logger.info(f"F.34 Source OpenStack VM server has been started")
-                if lib.wait_for_ostack_server_status(source_project_conn, i_source_server.id, 'ACTIVE') != "ACTIVE":
-                    args.logger.warning(f"F.34 Source OpenStack VM server has not become ACTIVE yet, trying again...")
-                    source_project_conn.compute.start_server(i_source_server_detail)
-                    args.logger.info(f"F.34 Source OpenStack VM server has been started again")
-                lib.log_or_assert(args, "F.34 Source OpenStack VM server started back (reached ACTIVE state)",
-                                  lib.wait_for_ostack_server_status(source_project_conn, i_source_server.id, 'ACTIVE') == "ACTIVE",
-                                  locals())
-
+                args.logger.info(f"F.34 Source OpenStack VM server (name:{i_source_server_detail.name}) requested to start")
+                i_src_server_state_active_check = True
 
         # start server in destination cloud
         i_destination_server = olib.create_dst_server(args,
@@ -239,6 +233,17 @@ def main(args):
 
         args.logger.info(f"F.41 Source OpenStack server name:{i_source_server_detail.name} migrated into destination one name:{i_destination_server.name} id:{i_destination_server.id}")
 
+        if i_src_server_state_active_check:
+            if lib.wait_for_ostack_server_status(source_project_conn, i_source_server.id, 'ACTIVE') != "ACTIVE":
+                args.logger.warning("F.42 Source OpenStack VM server has not become ACTIVE yet, trying again...")
+                source_project_conn.compute.start_server(i_source_server_detail)
+                args.logger.info(f"F.42 Source OpenStack VM server (name:{i_source_server_detail.name}) requested to start again")
+            if lib.wait_for_ostack_server_status(source_project_conn, i_source_server.id, 'ACTIVE') != "ACTIVE":
+                args.logger.error(f"F.42 Source OpenStack VM server (name:{i_source_server_detail.name}) has not become ACTIVE yet (after second start). " \
+                                  f"This situation is no longer asserted but needs manual admin inspection.")
+            else:
+                args.logger.info(f"F.42 Source OpenStack VM server (name:{i_source_server_detail.name}) became ACTIVE already.")
+
     # EXPLICIT OpenStack volume migration
     if args.explicit_volume_names:
         for i_source_volume_name in args.explicit_volume_names: