lib.py

""" OpenStack project migrator library """

import json
import re
import pprint
import time

import xmltodict
import paramiko
import openstack
from keystoneauth1.identity import v3
from keystoneauth1 import session


def wait_for_keypress(msg="Press Enter to continue..."):
    """ """
    return input("Press Enter to continue...")

def normalize_servers(servers):
    """ list of server names/IDs separated by space of comma returned as list of strings or None """
    if isinstance(servers, str) and servers:
        return servers.replace(","," ").split()
    return None

def trim_dict(dict_data, allowed_keys=None, denied_keys=None):
    """ transform input dictionary and filter its keys with allowed_keys and denied_keys sequences """
    int_allowed_keys = allowed_keys if allowed_keys else tuple()
    int_denied_keys = denied_keys if denied_keys else tuple()
    if int_allowed_keys:
        return {i_key: dict_data[i_key] for i_key in dict_data if i_key in int_allowed_keys}
    if int_denied_keys:
        return {i_key: dict_data[i_key] for i_key in dict_data if i_key not in int_denied_keys}
    return dict_data


def get_destination_network(source_network):
    """ LUT for networks """
    network_mapping = {
        # shared
        "78-128-250-pers-proj-net" :  "internal-ipv4-general-private",
        "147-251-115-pers-proj-net" : "internal-ipv4-general-private",
        "public-muni-v6-432" :        "external-ipv6-general-public",
        # external
        "public-muni-147-251-21-GROUP": "external-ipv4-general-public",
        "public-cesnet-78-128-250-PERSONAL": "external-ipv4-general-public",
        "public-cesnet-78-128-251-GROUP": "external-ipv4-general-public",
        "provider-public-cerit-sc-147-251-253": "external-ipv4-general-public",
        "public-muni-147-251-115-PERSONAL": "external-ipv4-general-public",
        "public-muni-147-251-124-GROUP": "external-ipv4-general-public",
        "public-cesnet-195-113-167-GROUP": "external-ipv4-general-public",
        "public-muni-147-251-11-128-254": "external-ipv4-general-public",
        "public-muni-CERIT-FI-147-251-88-132-254": "external-ipv4-general-public",
        "public-muni-CSIRT-MU-217-69-96-64-240": "external-ipv4-general-public",
        "public-muni-csirt-147-251-125-16-31": "external-ipv4-general-public",
        "provider-public-cerit-sc-147-251-254": "external-ipv4-general-public",
        # group project internal network
        "group-project-network": "group-project-network"
        }
    if source_network in network_mapping.keys():
        return network_mapping[source_network]
    return None

def get_destination_subnet(source_subnet):
    """ LUT for networks """
    subnet_mapping = {
        # TODO: shared

        # group project internal network
        "group-project-network-subnet": "group-project-network-subnet"
        }
    if source_subnet in subnet_mapping.keys():
        return subnet_mapping[source_subnet]
    return None

def get_destination_router(source_router):
    """ LUT for networks """
    router_mapping = {
        # TODO: shared

        # group project internal network
        "router": "group-project-router"
        }
    if source_router in router_mapping.keys():
        return router_mapping[source_router]
    return None


def get_destination_flavor(source_flavor):
    """ LUT for flavors """
    flavor_mapping = {
        #'eph.16cores-60ram' # nemusime resit neni pouzit u zadneho projektu v g1
        #'eph.8cores-30ram': 'c2.8core-30ram' # nemusime resit neni pouzit u zadneho projektu v g1
        #'eph.8cores-60ram': 'c3.8core-60ram' # nemusime resit neni pouzit u zadneho projektu v g1
        'hdn.cerit.large-35ssd-ephem': 'p3.4core-8ram', # nesedi velikost disku v G2 je 80 misto 35
        'hdn.cerit.large-ssd-ephem': 'p3.4core-8ram', # ok
        'hdn.cerit.medium-35ssd-ephem': 'p3.2core-4ram', # nesedi velikost disku v G2 je 80 misto 35
        'hdn.cerit.xxxlarge-ssd-ephem': 'p3.8core-60ram', # ok
        #'hdn.medium-ssd-ephem': # nemusime resit neni pouzit u zadneho projektu v g1
        'hpc.12core-64ram-ssd-ephem-500': 'c3.12core-64ram-ssd-ephem-500', # neni v G2 a je potreba
        'hpc.16core-128ram': 'c3.16core-128ram', # neni v G2 a je potreba
        'hpc.16core-256ram': 'c3.16core-256ram', # neni v G2 a je potreba
        'hpc.16core-32ram': 'c2.16core-30ram', # ok
        'hpc.16core-32ram-100disk': 'c3.16core-32ram-100disk', # neni v G2 a je potreba
        'hpc.16core-64ram-ssd-ephem': 'hpc.16core-64ram-ssd', # neni v G2 a je potreba
        'hpc.16core-64ram-ssd-ephem-500': 'p3.16core-60ram', # ok
        'hpc.18core-48ram': '', # neni v G2 a je potreba
        'hpc.18core-64ram-dukan': 'c2.24core-60ram', # nemusime resit
        'hpc.24core-96ram-ssd-ephem': 'hpc.24core-96ram-ssd', # nemusime resit
        'hpc.30core-128ram-ssd-ephem-500': 'c3.30core-128ram-ssd-ephem-500', # neni v G2 a je potreba
        'hpc.30core-256ram': 'c3.30core-256ram', # neni v G2 a je potreba
        'hpc.30core-64ram': 'c3.32core-60ram', # v G2 je o 2 CPU vic
        'hpc.4core-16ram-ssd-ephem': 'p3.4core-16ram', # ok
        'hpc.4core-16ram-ssd-ephem-500': 'p3.4core-16ram', #  ok
        'hpc.4core-4ram': 'e1.medium', # nemusime resit
        'hpc.8core-128ram': 'c3.8core-128ram', # neni v G2 a je potreba
        'hpc.8core-16ram': 'c2.8core-16ram', # ok
        'hpc.8core-16ram-ssd-ephem': 'p3.8core-16ram', # nemusime resit
        'hpc.8core-256ram': None, # nemusime resit
        'hpc.8core-32ram-dukan': 'c2.8core-30ram', # nemusime resit
        'hpc.8core-32ram-ssd-ephem': 'p3.8core-30ram', # ok
        'hpc.8core-32ram-ssd-rcx-ephem': 'p3.8core-30ram', # ok
        'hpc.8core-64ram-ssd-ephem-500': 'p3.8core-60ram', # ok
        'hpc.8core-8ram': 'e1.1xlarge', # v G2 je o 20 GB mensi disk
        'hpc.hdh-ephem': 'hpc.hdh', # neni a je potreba
        'hpc.hdn.30core-128ram-ssd-ephem-500': 'c3.hdn.30core-128ram-ssd-ephem-500', # neni potreba
        'hpc.hdn.4core-16ram-ssd-ephem-500': 'p3.4core-16ram', # neni potreba
        #'hpc.ics-gladosag-full': 'c3.ics-gladosag-full', # neni potreba
        'hpc.large': 'g2.3xlarge', # ok
        'hpc.medium': 'c2.8core-30ram', # ok
        'hpc.small': 'c2.4core-16ram', # ok
        'hpc.xlarge': None, # neni v G2
        'hpc.xlarge-memory': 'c3.xlarge-memory', # neni v G2
        'standard.16core-32ram': 'g2.2xlarge', # ok
        'standard.20core-128ram': 'e1.20core-128ram', # neni potreba
        'standard.20core-256ram': 'e1.20core-256ram', # neni v G2
        'standard.2core-16ram': 'c3.2core-16ram', # ok
        'standard.large': 'e1.large', # ok pripadne jeste c3.4core-8ram
        'standard.medium': 'e1.medium', # o 2 vice CPU
        'standard.memory': 'c3.2core-30ram', # pripadne i c2.2core-30ram
        'standard.one-to-many': 'c3.24core-60ram', # v G2 je o 4 vice CPU
        'standard.small': 'e1.small', # 2x vice ram a CPU u G2
        'standard.tiny': 'e1.tiny', # 2x vice ram a CPU u G2
        'standard.xlarge': 'e1.2xlarge', # o 4 vice CPU G2
        'standard.xlarge-cpu': 'e1.2xlarge', # ok
        'standard.xxlarge': 'c2.8core-30ram', # ok
        'standard.xxxlarge': 'c3.8core-60ram'  # ok
    }
    assert source_flavor in flavor_mapping, "Source flavor can be mapped to destination one"
    assert flavor_mapping[source_flavor], "Source flavor mapping is not valid"
    return flavor_mapping[source_flavor]

def normalize_table_data_field(data_field):
    """ normalize single data field (single data insert) """
    int_dict = {}
    i_name_key = '@name'
    for i_data_field_item in data_field:
        i_value_key = [ i_k for i_k in i_data_field_item.keys() if i_k != i_name_key][0]
        int_dict[i_data_field_item[i_name_key]] = i_data_field_item[i_value_key]
    return int_dict

def normalize_table_data(data):
    """ normalize whole table data """
    int_list = []
    for i_data_field in data:
        int_list.append(normalize_table_data_field(i_data_field['field']))
    return int_list

def get_migrated_resource_name(args, name):
    """ translate original name to destination one """
    return f"{args.destination_entity_prefix}{name}"

def get_openrc(file_handle):
    """ parse and return OpenRC file """
    openrc_vars = {}

    for line in file_handle:
        match = re.match(r'^export (\w+)=(.+)$', line.strip())
        if match:
            openrc_vars[match.group(1)] = match.group(2).strip('"')
    return openrc_vars


def get_ostack_connection(openrc_vars):
    """ """
    auth_args = {
        'auth_url': openrc_vars.get('OS_AUTH_URL'),
        'username': openrc_vars.get('OS_USERNAME'),
        'password': openrc_vars.get('OS_PASSWORD'),
        'project_name': openrc_vars.get('OS_PROJECT_NAME'),
        'project_domain_name': openrc_vars.get('OS_PROJECT_DOMAIN_NAME'),
        'user_domain_name': openrc_vars.get('OS_USER_DOMAIN_NAME'),
        'project_domain_id': openrc_vars.get('OS_PROJECT_DOMAIN_ID'),
        'user_domain_id': openrc_vars.get('OS_USER_DOMAIN_ID'),
    }
    connection_args = {
        'compute_api_version': openrc_vars.get('OS_COMPUTE_API_VERSION'),
        'identity_api_version': openrc_vars.get('OS_IDENTITY_API_VERSION'),
        'volume_api_version': openrc_vars.get('OS_VOLUME_API_VERSION')
    }
    auth = v3.Password(**auth_args)
    ostack_sess = session.Session(auth=auth)
    ostack_conn = openstack.connection.Connection(session=ostack_sess, **connection_args)
    return ostack_conn

def get_ostack_project(ostack_connection, project_name):
    project = None
    for i_project in ostack_connection.list_projects():
        if i_project.name == project_name:
            project = i_project
    return project

def get_ostack_project_type(ostack_connection, project):
    """ detect project type, return 'group' / 'personal' / 'other' """
    if project.name in [ i_user.name for i_user in ostack_connection.list_users() ]:
        return "personal"
    return "group"

def get_ostack_project_security_groups(ostack_connection, project=None):
    security_groups = []
    if project:
        for i_security_group in ostack_connection.network.security_groups():
            if i_security_group.tenant_id == project.id:
                security_groups.append(i_security_group)
        return security_groups
    return tuple(ostack_connection.network.security_groups())

def get_ostack_project_keypairs(ostack_connection, project=None):
    return ostack_connection.list_keypairs()
def get_ostack_project_keypairs2(ostack_connection, project=None):
    return list(ostack_connection.compute.keypairs())


def get_ostack_project_servers(ostack_connection, project=None):
    return tuple(ostack_connection.compute.servers())

def get_ostack_project_volumes(ostack_connection, project=None):
    return ostack_connection.block_store.volumes()

def get_ostack_project_flavors(ostack_connection, project=None):
    return tuple(ostack_connection.compute.flavors())

def get_resource_details(resources):
    """ inspect resources """
    for i_resource in resources:
        print(i_resource)
        pprint.pprint(i_resource)

def remote_cmd_exec(hostname, username, key_filename, command):
    """ executes remote command, returs stdout, stderr and exit-code or Exception """
    # Create SSH client
    ssh_client = paramiko.SSHClient()
    # Automatically add untrusted hosts
    ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ecode = None

    try:
        # Connect to the remote host
        pkey = paramiko.RSAKey.from_private_key_file(key_filename)
        ssh_client.connect(hostname, username=username, pkey=pkey, look_for_keys=False)

        # Execute the command, read the output and close
        stdin, stdout, stderr = ssh_client.exec_command(command)
        output = stdout.read().decode().strip()
        error = stderr.read().decode().strip()
        ecode = stdout.channel.recv_exit_status()
        ssh_client.close()

        return output, error, ecode

    except Exception as e:
        print("Error:", e)
        return None, None, e

def get_ceph_client_name(args, ceph_src_pool_name, ceph_dst_pool_name=None):
    int_pool_name = ceph_dst_pool_name if ceph_dst_pool_name else ceph_src_pool_name

    return "client.cinder" if int_pool_name in (args.source_ceph_cinder_pool_name, args.source_ceph_ephemeral_pool_name,) else "client.migrator"

def ceph_rbd_images_list(args, pool_name):
    """ """
    stdout, stderr, ecode = remote_cmd_exec(args.ceph_migrator_host,
                                            args.ceph_migrator_user,
                                            args.ceph_migrator_sshkeyfile.name,
                                            f"/root/migrator/ceph-rbd-images-list.sh {pool_name}")
    assert stdout, f"RBD pool ({pool_name}) images received successfully (non-empty RBD list)"
    assert ecode == 0, f"RBD pool ({pool_name}) images received successfully (ecode)"
    return stdout.splitlines()

def ceph_rbd_image_info(args, pool_name, rbd_image_name):
    """ get ceph RBD image information """
    ceph_client_name = get_ceph_client_name(args, pool_name)
    stdout, stderr, ecode = remote_cmd_exec(args.ceph_migrator_host,
                                            args.ceph_migrator_user,
                                            args.ceph_migrator_sshkeyfile.name,
                                            f"CEPH_USER={ceph_client_name} /root/migrator/ceph-rbd-image-info.sh {pool_name} {rbd_image_name}")
    return json.loads(stdout), stderr, ecode



def ceph_rbd_image_exists(args, pool_name, rbd_image_name):
    """ detect whether RBD image {pool_name}/{rbd_image_name} exists """
    ceph_client_name = get_ceph_client_name(args, pool_name)
    stdout, stderr, ecode = remote_cmd_exec(args.ceph_migrator_host,
                                            args.ceph_migrator_user,
                                            args.ceph_migrator_sshkeyfile.name,
                                            f"CEPH_USER={ceph_client_name} /root/migrator/ceph-rbd-image-exists.sh {pool_name} {rbd_image_name}")
    return stdout.splitlines(), stderr, ecode

def ceph_rbd_image_delete(args, pool_name, rbd_image_name):
    """ delete RBD image {pool_name}/{rbd_image_name} """
    ceph_client_name = get_ceph_client_name(args, pool_name)
    stdout, stderr, ecode = remote_cmd_exec(args.ceph_migrator_host,
                                            args.ceph_migrator_user,
                                            args.ceph_migrator_sshkeyfile.name,
                                            f"CEPH_USER={ceph_client_name} /root/migrator/ceph-rbd-image-delete.sh {pool_name} {rbd_image_name}")
    return stdout.splitlines(), stderr, ecode

def ceph_rbd_image_flatten(args, pool_name, rbd_image_name):
    """ flatten RBD image {pool_name}/{rbd_image_name} """
    ceph_client_name = get_ceph_client_name(args, pool_name)
    stdout, stderr, ecode = remote_cmd_exec(args.ceph_migrator_host,
                                            args.ceph_migrator_user,
                                            args.ceph_migrator_sshkeyfile.name,
                                            f"CEPH_USER={ceph_client_name} /root/migrator/ceph-rbd-image-flatten.sh {pool_name} {rbd_image_name}")
    return stdout.splitlines(), stderr, ecode

def ceph_rbd_image_clone(args, src_pool_name, src_rbd_image_name, src_rbd_image_snapshot_name,
                         dst_pool_name, dst_rbd_image_name):
    """ clone RBD image {src_pool_name}/{src_rbd_image_name}@{src_rbd_image_snapshot_name} -> {dst_pool_name}/{dst_rbd_image_name}"""
    ceph_client_name = get_ceph_client_name(args, src_pool_name, dst_pool_name)
    stdout, stderr, ecode = remote_cmd_exec(args.ceph_migrator_host,
                                            args.ceph_migrator_user,
                                            args.ceph_migrator_sshkeyfile.name,
                                            f"CEPH_USER={ceph_client_name} /root/migrator/ceph-rbd-image-clone.sh {src_pool_name} {src_rbd_image_name} {src_rbd_image_snapshot_name} {dst_pool_name} {dst_rbd_image_name}")
    return stdout.splitlines(), stderr, ecode

def ceph_rbd_image_copy(args, src_pool_name, src_rbd_image_name, dst_pool_name, dst_rbd_image_name):
    """ copy RBD image {src_pool_name}/{src_rbd_image_name} -> {dst_pool_name}/{dst_rbd_image_name}"""
    ceph_client_name = get_ceph_client_name(args, src_pool_name, dst_pool_name)
    cmd = f"CEPH_USER={ceph_client_name} /root/migrator/ceph-rbd-image-copy.sh {src_pool_name} {src_rbd_image_name} {dst_pool_name} {dst_rbd_image_name}"
    stdout, stderr, ecode = remote_cmd_exec(args.ceph_migrator_host,
                                            args.ceph_migrator_user,
                                            args.ceph_migrator_sshkeyfile.name,
                                            cmd)
    return stdout.splitlines(), stderr, ecode


def ceph_rbd_image_snapshot_exists(args, pool_name, rbd_image_name, rbd_image_snapshot_name):
    """ detect whether RBD image snapshot {pool_name}/{rbd_image_name}@{rbd_image_snapshot_name} exists """
    ceph_client_name = get_ceph_client_name(args, pool_name)
    stdout, stderr, ecode = remote_cmd_exec(args.ceph_migrator_host,
                                            args.ceph_migrator_user,
                                            args.ceph_migrator_sshkeyfile.name,
                                            f"CEPH_USER={ceph_client_name} /root/migrator/ceph-rbd-image-snapshot-exists.sh {pool_name} {rbd_image_name} {rbd_image_snapshot_name}")
    return stdout.splitlines(), stderr, ecode

def ceph_rbd_image_snapshot_create(args, pool_name, rbd_image_name, rbd_image_snapshot_name):
    """ create RBD image snapshot {pool_name}/{rbd_image_name}@{rbd_image_snapshot_name} """
    ceph_client_name = get_ceph_client_name(args, pool_name)
    stdout, stderr, ecode = remote_cmd_exec(args.ceph_migrator_host,
                                            args.ceph_migrator_user,
                                            args.ceph_migrator_sshkeyfile.name,
                                            f"CEPH_USER={ceph_client_name} /root/migrator/ceph-rbd-image-snapshot-create.sh {pool_name} {rbd_image_name} {rbd_image_snapshot_name}")
    return stdout.splitlines(), stderr, ecode

def ceph_rbd_image_snapshot_delete(args, pool_name, rbd_image_name, rbd_image_snapshot_name):
    """ delete RBD image snapshot {pool_name}/{rbd_image_name}@{rbd_image_snapshot_name} """
    ceph_client_name = get_ceph_client_name(args, pool_name)
    stdout, stderr, ecode = remote_cmd_exec(args.ceph_migrator_host,
                                            args.ceph_migrator_user,
                                            args.ceph_migrator_sshkeyfile.name,
                                            f"CEPH_USER={ceph_client_name} /root/migrator/ceph-rbd-image-snapshot-delete.sh {pool_name} {rbd_image_name} {rbd_image_snapshot_name}")
    return stdout.splitlines(), stderr, ecode


def assert_entity_ownership(entities, project):
    """ """
    for i_entity in entities:
        assert i_entity.project_id == project.id, f"Entity belongs to expected project (id: {project.id})"

def get_source_keypairs(args):
    """ """
    reply_stdout, reply_stderr, reply_ecode = remote_cmd_exec(args.ceph_migrator_host,
                                                              args.ceph_migrator_user,
                                                              args.ceph_migrator_sshkeyfile.name,
                                                              f"cat {args.source_keypair_xml_dump_file}")
    assert reply_ecode == 0, "Keypairs received"
    table_dictdata = xmltodict.parse(reply_stdout)
    table_data_dictdata = table_dictdata['mysqldump']['database']['table_data']['row']
    return normalize_table_data(table_data_dictdata)

def get_source_keypair(keypairs, keypair_name, user_id):
    """ """
    keypairs_selected = [ i_keypair for i_keypair in keypairs if i_keypair.get("name", "") == keypair_name and i_keypair.get("user_id", "") == user_id ]
    if keypairs_selected:
        return keypairs_selected[0]
    return None

def create_keypair(args, ostack_connection, keypair):
    """ create openstack keypair object """
    return ostack_connection.compute.create_keypair(name=get_migrated_resource_name(args, keypair['name']),
                                                    public_key=keypair['public_key'], type=keypair['type'])

def create_security_group(args, ostack_connection, security_group, project):
    """ create openstack security group """
    int_sg = ostack_connection.network.create_security_group(name=get_migrated_resource_name(args, security_group.name),
                                                             description=security_group.description,
                                                             project_id=project.id)
    #pprint.pprint(int_sg)

    for i_rule in security_group.security_group_rules:
        i_mod_rule = {i_k: i_rule[i_k] for i_k in i_rule if i_k not in ['id', 'project_id', 'tenant_id', 'revision_number', 'updated_at', 'created_at', 'tags', 'standard_attr_id', 'normalized_cidr']}
        i_mod_rule['security_group_id'] = int_sg.id
        i_mod_rule['project_id'] = project.id
        i_mod_rule = {i_k: i_mod_rule[i_k] for i_k in i_mod_rule if i_mod_rule[i_k] is not None}
        #pprint.pprint(i_rule)
        #pprint.pprint(i_mod_rule)
        try:
            ostack_connection.network.create_security_group_rule(**i_mod_rule)
        except openstack.exceptions.ConflictException as ex:
            # TODO: analyze whether Conflicts we have seen meat that security group role IS COMPLETELY identical
            # Alternative solution would be to remove rules after creation and add specific ones
            pass

    return int_sg

def log_or_assert(args, msg, condition, trace_details=None):
    """ log, assert, dump state """
    if not condition:
        with open(args.exception_trace_file, "w") as file:
            file.write(f"{msg}\n{pprint.pformat(trace_details)}\n\n{locals()}\n")
    assert condition, msg
    args.logger.info(msg)


def wait_for_ostack_server_status(ostack_connection, server_name_or_id, server_status, timeout=120):
    """ """
    int_start_timestamp = time.time()
    int_server = ostack_connection.compute.find_server(server_name_or_id)
    int_server_status = None
    while True:
        if time.time() > (int_start_timestamp + timeout):
            break
        int_server_status = ostack_connection.compute.find_server(int_server.id).status
        if int_server_status == server_status:
            break

    return int_server_status

def wait_for_ostack_volume_status(ostack_connection, volume_name_or_id, volume_status, timeout=120):
    """ """
    int_start_timestamp = time.time()
    int_volume = ostack_connection.block_storage.find_volume(volume_name_or_id)
    int_volume_status = None
    while True:
        if time.time() > (int_start_timestamp + timeout):
            break
        int_volume_status = ostack_connection.block_storage.find_volume(int_volume.id).status
        if int_volume_status == volume_status:
            break

    return int_volume_status

def server_detect_floating_address(server):
    """ return True if server has attached floating IP address otherwise False """
    for _, i_ip_details in server.addresses.items():
        for i_ip_detail in i_ip_details:
            if str(i_ip_detail.get('version')) == '4' and i_ip_detail.get('OS-EXT-IPS:type') == 'floating':
                return True
    return False

def get_server_floating_ip_port(ostack_connection, server):
    """ set server's port where to put FIP, otherwise None """
    for i_port in ostack_connection.network.ports(device_id=server.id):
        for i_port_ip in i_port.fixed_ips:
            for i_ip_prefix in ('192.', '10.', '172.'):
                if str(i_port_ip.get('ip_address')).startswith(i_ip_prefix):
                    return i_port
    return None