diff --git a/ci/lib.py b/ci/lib.py
index 0eab6d78e1df242da87a80ca104c151b68eee754..6f24132196aa0828c156acb3c020a1927fb57022 100644
--- a/ci/lib.py
+++ b/ci/lib.py
@@ -1,5 +1,6 @@
""" OpenStack project migrator library """
+import copy
import json
import re
import pprint
@@ -403,29 +404,58 @@ def create_keypair(args, ostack_connection, keypair):
return ostack_connection.compute.create_keypair(name=get_migrated_resource_name(args, keypair['name']),
public_key=keypair['public_key'], type=keypair['type'])
-def create_security_group(args, ostack_connection, security_group, project):
- """ create openstack security group """
- int_sg = ostack_connection.network.create_security_group(name=get_migrated_resource_name(args, security_group.name),
- description=security_group.description,
- project_id=project.id)
- #pprint.pprint(int_sg)
-
- for i_rule in security_group.security_group_rules:
- i_mod_rule = {i_k: i_rule[i_k] for i_k in i_rule if i_k not in ['id', 'project_id', 'tenant_id', 'revision_number', 'updated_at', 'created_at', 'tags', 'standard_attr_id', 'normalized_cidr']}
+def create_security_groups(args, src_ostack_conn, dst_ostack_conn, src_security_group, dst_project, recursion_stack=None):
+ """ create openstack security group[s] """
+ int_recursion_stack = {} if recursion_stack is None else recursion_stack
+ int_sg = dst_ostack_conn.network.create_security_group(name=get_migrated_resource_name(args, src_security_group.name),
+ description=f"{src_security_group.description}, g1-to-g2-migrated(g1-id:{src_security_group.id})",
+ project_id=dst_project.id)
+ int_recursion_stack[src_security_group.id] = int_sg.id
+
+ for i_rule in src_security_group.security_group_rules:
+ # browse security group rules
+ i_mod_rule = trim_dict(i_rule, denied_keys=['id', 'project_id', 'tenant_id', 'revision_number', 'updated_at', 'created_at', 'tags', 'standard_attr_id', 'normalized_cidr'])
i_mod_rule['security_group_id'] = int_sg.id
- i_mod_rule['project_id'] = project.id
+ i_mod_rule['project_id'] = dst_project.id
i_mod_rule = {i_k: i_mod_rule[i_k] for i_k in i_mod_rule if i_mod_rule[i_k] is not None}
- #pprint.pprint(i_rule)
- #pprint.pprint(i_mod_rule)
+ if i_mod_rule.get('remote_group_id') is not None:
+ if i_mod_rule['remote_group_id'] in int_recursion_stack:
+ # keep reference to itself or known (already created) SGs
+ i_mod_rule['remote_group_id'] = int_recursion_stack[i_mod_rule['remote_group_id']]
+ # get linked source SG
+ elif _src_sg := src_ostack_conn.network.find_security_group(i_mod_rule['remote_group_id']):
+ if _dst_sg := dst_ostack_conn.network.find_security_group(get_migrated_resource_name(args, _src_sg.name),
+ project_id=dst_project.id):
+ i_mod_rule['remote_group_id'] = _dst_sg.id
+ else:
+ int_linked_sg = create_security_groups(args, src_ostack_conn, dst_ostack_conn,
+ _src_sg, dst_project,
+ copy.deepcopy(int_recursion_stack))
+ i_mod_rule['remote_group_id'] = int_linked_sg.id
try:
- ostack_connection.network.create_security_group_rule(**i_mod_rule)
+ dst_ostack_conn.network.create_security_group_rule(**i_mod_rule)
except openstack.exceptions.ConflictException as ex:
- # TODO: analyze whether Conflicts we have seen meat that security group role IS COMPLETELY identical
- # Alternative solution would be to remove rules after creation and add specific ones
pass
return int_sg
+def duplicate_ostack_project_security_groups(args, src_ostack_conn, dst_ostack_conn, src_project, dst_project):
+ """ duplicate all projects's openstack security group[s] """
+
+ src_project_security_groups = tuple(src_ostack_conn.network.security_groups(project_id=src_project.id))
+
+ for i_src_security_group in src_project_security_groups:
+ j_dst_security_group_found = False
+ for j_dst_security_group in tuple(dst_ostack_conn.network.security_groups(project_id=dst_project.id)):
+ if get_migrated_resource_name(args, i_src_security_group.name) == j_dst_security_group.name and \
+ i_src_security_group.id in j_dst_security_group.description:
+ j_dst_security_group_found = True
+ if not j_dst_security_group_found:
+ create_security_groups(args, src_ostack_conn, dst_ostack_conn, i_src_security_group, dst_project)
+
+ return src_project_security_groups, tuple(dst_ostack_conn.network.security_groups(project_id=dst_project.id))
+
+
def log_or_assert(args, msg, condition, trace_details=None):
""" log, assert, dump state """
if not condition:
@@ -666,5 +696,3 @@ def migrate_rbd_image(args, server_block_device_mapping):
"G.17 Source OpenStack VM RBD image snapshot does not exist anymore " \
f"{server_block_device_mapping['source']['ceph_pool_name']}/{source_server_rbd_image}@{source_rbd_image_snapshot_name}",
ecode != 0, locals())
-
-
diff --git a/ci/project-migrator.py b/ci/project-migrator.py
index cdb1df434a390159e32dcdca7ade99c073904fbf..926cbea2e9496200912d88c82c60d09895a00813 100755
--- a/ci/project-migrator.py
+++ b/ci/project-migrator.py
@@ -2,15 +2,15 @@
"""
OpenStack project multicloud migrator
+Tool performs OpenStack workflow migration from single OpenStack cloud to another one.
+Block storage is transferred on external node using ceph low-level commands.
Usage example:
* ./project-migrator.py --source-openrc ~/c/prod-einfra_cz_migrator.sh.inc
--destination-openrc ~/c/g2-prod-brno-einfra_cz_migrator.sh.inc
--project-name meta-cloud-new-openstack
--validation-a-source-server-id <>
- --ceph-migrator-sshkeyfile $HOME/.ssh/id_rsa.LenovoThinkCentreE73
- * ./project-migrator.py --source-openrc ~/c/prod-einfra_cz_migrator.sh.inc --destination-openrc ~/c/g2-prod-brno-einfra_cz_migrator.sh.inc --project-name meta-cloud-new-openstack --validation-a-source-server-id <> --ceph-migrator-sshkeyfile $HOME/.ssh/id_rsa.LenovoThinkCentreE73 --explicit-server-names freznicek-rook-internal-external-20-worker-1
- *
+ --ceph-migrator-sshkeyfile ~/.ssh/id_rsa.g1-g2-ostack-cloud-migration
"""
import argparse
@@ -24,7 +24,7 @@ import sys
import lib
def main(args):
- """ """
+ """ main project migration loop """
# connect to source cloud
source_migrator_openrc = lib.get_openrc(args.source_openrc)
source_migrator_conn = lib.get_ostack_connection(source_migrator_openrc)
@@ -106,6 +106,9 @@ def main(args):
destination_fip_network = destination_project_conn.network.find_network(args.destination_ipv4_external_network)
lib.log_or_assert(args, "E.31 Destination cloud FIP network detected", destination_fip_network)
+ lib.duplicate_ostack_project_security_groups(args, source_project_conn, destination_project_conn,
+ source_project, destination_project)
+ args.logger.info("E.40 Destination OpenStack project security groups duplicated")
args.logger.info("F.0 Main looping started")
args.logger.info(f"F.0 Source VM servers: {[ i_source_server.name for i_source_server in source_project_servers]}")
@@ -126,7 +129,11 @@ def main(args):
args.logger.info(f"F.1 server migration skipped - name:{i_source_server_detail.name} as equivalent VM exists in destination cloud (name: {i_destination_server_detail.name})")
continue
- args.logger.info(f"F.1 server migration started - name:{i_source_server_detail.name}, id:{i_source_server_detail.id}, keypair: {i_source_server_detail.key_name}, flavor: {i_source_server_detail.flavor}, sec-groups:{i_source_server_detail.security_groups}, root_device_name: {i_source_server_detail.root_device_name}, block_device_mapping: {i_source_server_detail.block_device_mapping}, attached-volumes: {i_source_server_detail.attached_volumes}")
+ args.logger.info(f"F.1 server migration started - name:{i_source_server_detail.name}, id:{i_source_server_detail.id}, " \
+ f"keypair: {i_source_server_detail.key_name}, flavor: {i_source_server_detail.flavor}, " \
+ f"sec-groups:{i_source_server_detail.security_groups}, root_device_name: {i_source_server_detail.root_device_name}, " \
+ f"block_device_mapping: {i_source_server_detail.block_device_mapping}, " \
+ f"attached-volumes: {i_source_server_detail.attached_volumes}")
# network, subnet detection, TODO: better
i_source_server_network_names = i_source_server_detail.addresses.keys()
@@ -136,9 +143,12 @@ def main(args):
if not i_destination_network_name and args.destination_group_project_network_name != "":
# if network is not mapped use network provided from switch --destination-group-project-network-name
i_destination_network_name = args.destination_group_project_network_name
- lib.log_or_assert(args, f"F.2 Source to Destination network mapping succeeeded ({i_source_network_name}->{i_destination_network_name}). Read --destination-group-project-network-name description for more details", i_destination_network_name)
+ lib.log_or_assert(args,
+ f"F.2 Source to Destination network mapping succeeeded ({i_source_network_name}->{i_destination_network_name}). " \
+ f"Read --destination-group-project-network-name description for more details",
+ i_destination_network_name)
- i_destination_network = destination_project_conn.network.find_network(i_destination_network_name)
+ i_destination_network = destination_project_conn.network.find_network(i_destination_network_name, project_id=destination_project.id)
lib.log_or_assert(args, f"F.3 Destination network exists ({i_destination_network})", i_destination_network)
i_destination_server_networks.append(i_destination_network)
@@ -156,7 +166,8 @@ def main(args):
lib.log_or_assert(args, f"F.7 Source OpenStack server keypair found ({i_source_server_keypair['name']})", i_source_server_keypair)
i_destination_server_keypair = None
- if i_destination_server_keypairs := [i_keypair for i_keypair in destination_project_conn.list_keypairs() if i_keypair.name == lib.get_migrated_resource_name(args, i_source_server_detail.key_name)]:
+ if i_destination_server_keypairs := [i_keypair for i_keypair in destination_project_conn.list_keypairs()
+ if i_keypair.name == lib.get_migrated_resource_name(args, i_source_server_detail.key_name)]:
i_destination_server_keypair = i_destination_server_keypairs[0]
lib.log_or_assert(args, f"F.8 Destination OpenStack server keypair found already ({i_destination_server_keypair.name})", i_destination_server_keypair)
else:
@@ -164,12 +175,9 @@ def main(args):
args.logger.info("F.8 Destination OpenStack server keypair created")
lib.log_or_assert(args, f"F.9 Destination OpenStack server keypair exists ({i_destination_server_keypair.name})", i_destination_server_keypair)
- # security group
- #source_project_security_groups = get_ostack_project_security_groups(source_project_conn, source_project)
- #destination_project_security_groups = get_ostack_project_security_groups(destination_project_conn, destination_project)
+ # server security group
i_destination_server_security_groups=[]
-
- for i_source_server_security_group_name in set([i_sg['name'] for i_sg in i_source_server_detail.security_groups]):
+ for i_source_server_security_group_name in {i_sg['name'] for i_sg in i_source_server_detail.security_groups}:
i_source_server_security_group = source_project_conn.network.find_security_group(i_source_server_security_group_name, project_id=source_project.id)
i_destination_server_security_group = None
if i_destination_server_security_group := destination_project_conn.network.find_security_group(lib.get_migrated_resource_name(args, i_source_server_security_group.name),
@@ -177,8 +185,9 @@ def main(args):
lib.log_or_assert(args, f"F.10 Destination OpenStack server security group found already ({i_destination_server_security_group.name})",
i_destination_server_security_group)
else:
- args.logger.info("F.10 Destination OpenStack server matching security group not found, gets created.")
- i_destination_server_security_group = lib.create_security_group(args, destination_project_conn, i_source_server_security_group, destination_project)
+ args.logger.info("F.10 Destination OpenStack server matching security group not found and gets created.")
+ i_destination_server_security_group = lib.create_security_groups(args, source_project_conn, destination_project_conn,
+ i_source_server_security_group, destination_project)
lib.log_or_assert(args, f"F.10 Destination OpenStack server security group created ({i_destination_server_security_group.name})",
i_destination_server_security_group)
@@ -225,7 +234,7 @@ def main(args):
# get rbd image info / size
i_source_ceph_ephemeral_rbd_image_data = lib.ceph_rbd_image_info(args, args.source_ceph_ephemeral_pool_name,
- i_source_ceph_ephemeral_rbd_image)
+ i_source_ceph_ephemeral_rbd_image)
lib.log_or_assert(args, f"F.24 Source OpenStack ceph RBD image information received {i_source_ceph_ephemeral_rbd_image_data}",
i_source_ceph_ephemeral_rbd_image_data and 'size' in i_source_ceph_ephemeral_rbd_image_data)
i_source_ceph_ephemeral_rbd_image_size = math.ceil(i_source_ceph_ephemeral_rbd_image_data['size'] / 1024 / 1024 / 1024)
@@ -264,7 +273,8 @@ def main(args):
for i_destination_server_block_device_mapping in i_server_block_device_mappings:
i_new_volume_args = {'name': i_destination_server_block_device_mapping['destination']['volume_name'],
'size': i_destination_server_block_device_mapping['destination']['volume_size'],
- 'description': f"{i_destination_server_block_device_mapping['destination']['volume_description']}, g1-to-g2-migrated(g1-id:{i_destination_server_block_device_mapping['source']['volume_id']})"}
+ 'description': f"{i_destination_server_block_device_mapping['destination']['volume_description']}, " \
+ f"g1-to-g2-migrated(g1-id:{i_destination_server_block_device_mapping['source']['volume_id']})"}
# TO BE REVISED: this seems to be the only way how to create bootable volume using openstacksdk
if i_destination_server_block_device_mapping['destination']['volume_bootable']:
i_new_volume_args['imageRef'] = destination_image.id
@@ -320,21 +330,26 @@ def main(args):
#pprint.pprint(i_destination_server_args)
i_destination_server = destination_project_conn.compute.create_server(**i_destination_server_args)
- lib.log_or_assert(args, "F.37 Destination OpenStack server is created", i_destination_server, locals())
+ lib.log_or_assert(args,
+ f"F.37 Destination OpenStack server (name:{i_destination_server.name}) is created",
+ i_destination_server, locals())
i_destination_server = destination_project_conn.compute.wait_for_server(i_destination_server)
- lib.log_or_assert(args, "F.38 Destination OpenStack server got ACTIVE",
- i_destination_server.status == 'ACTIVE', locals())
+ lib.log_or_assert(args,
+ f"F.38 Destination OpenStack server (name:{i_destination_server.name}) got ACTIVE",
+ i_destination_server.status == 'ACTIVE', locals())
# add security groups to the destination server (if missing)
- for i_destination_server_security_group_id, i_destination_server_security_group_name in set([(i_destination_server_security_group.id, i_destination_server_security_group.name) for i_destination_server_security_group in i_destination_server_security_groups]):
+ for i_destination_server_security_group_id, i_destination_server_security_group_name in {(i_destination_server_security_group.id, i_destination_server_security_group.name) for i_destination_server_security_group in i_destination_server_security_groups}:
if {'name': i_destination_server_security_group_name } not in i_destination_server.security_groups:
destination_project_conn.add_server_security_groups(i_destination_server.id, i_destination_server_security_group_id)
if i_source_server_has_fip:
# add FIP as source VM has it
i_destination_server_fip = destination_project_conn.network.create_ip(floating_network_id=destination_fip_network.id)
- lib.log_or_assert(args, "F.39 Destination OpenStack server FIP is created", i_destination_server_fip, locals())
+ lib.log_or_assert(args, f"F.39 Destination OpenStack server (name:{i_destination_server.name}) FIP is created ({i_destination_server_fip.floating_ip_address})",
+ i_destination_server_fip, locals())
i_destination_server_port = lib.get_server_floating_ip_port(destination_project_conn, i_destination_server)
- lib.log_or_assert(args, "F.40 Destination OpenStack server FIP port is detected", i_destination_server_port, locals())
+ lib.log_or_assert(args, f"F.40 Destination OpenStack server (name:{i_destination_server.name}) FIP port is detected",
+ i_destination_server_port, locals())
destination_project_conn.network.add_ip_to_port(i_destination_server_port, i_destination_server_fip)
args.logger.info(f"F.41 Source OpenStack server name:{i_source_server_detail.name} migrated into destination one name:{i_destination_server.name} id:{i_destination_server.id}")