diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1fb6080d1b98d30cd741f67684249fa437da3959..14efb635d9b9b8f320a14601ca8f9069c08fe879 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ stages: - test - migration + - after-migration image: registry.gitlab.ics.muni.cz:443/cloud/g2/common-cloud-entities:1.0.2 @@ -56,3 +57,20 @@ ostack-einfra_cz-project-flavor-check: resource_group: migration-flavor-check script: - ci/project-flavor-migration-check.py --source-openrc="${SRC_CLOUD_OSTACK_RC_FILE}" --destination-openrc="${DST_CLOUD_OSTACK_RC_FILE}" --project-name="${PROJECT_NAME}" + +ostack-einfra_cz-trigger-communication-generation: + <<: *migration-job + stage: after-migration + resource_group: communication-generation + when: manual + variables: + TEMPLATE_PATH: "howtos/email_tool/g1-g2-migration/templates/mail-final_project-migrated" + TRIGGER_URL: "https://gitlab.ics.muni.cz/api/v4/projects/2804/trigger/pipeline" + script: + - ci/generate-data-for-communication.py --source-openrc="${SRC_CLOUD_OSTACK_RC_FILE}" --destination-openrc="${DST_CLOUD_OSTACK_RC_FILE}" --project-name="${PROJECT_NAME}" --signature="${GITLAB_USER_NAME}" --expiration="${PROJECT_EXPIRATION}" + - zip -P ${ZIP_DATA_ENCRYTPION_PASSWORD} data.zip data.csv servers.csv + - BASE64_ZIP_FILE=$(base64 -w 0 data.zip) + - RESPONSE=$(curl -X POST -F token=${TRIGGER_TOKEN_KB} -F ref=master -F "variables[TEMPLATE]=${TEMPLATE_PATH}" -F "variables[BASE64_DATA]=${BASE64_ZIP_FILE}" ${TRIGGER_URL}) + - PIPELINE_ID=$(echo $RESPONSE | jq -r '.id') + - PIPELINE_URL="https://gitlab.ics.muni.cz/cloud/knowledgebase/-/pipelines/$PIPELINE_ID" + - echo "Pipeline triggered successfully. View it at $PIPELINE_URL" diff --git a/ci/generate-data-for-communication.py b/ci/generate-data-for-communication.py new file mode 100755 index 0000000000000000000000000000000000000000..bf221888c3df001d2d0f3ff9eb4533aafb260780 --- /dev/null +++ b/ci/generate-data-for-communication.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +""" +OpenStack project data extractor. + +Tool which provides OpenStack project data, which is used to generate email to user. +It's used in ostack-einfra_cz-trigger-communication-generation pipeline job to generate csv files, which are send to pipiline in kb generate-communication. + + +Tool relies on main libraries: + * openstacksdk for OpenStack management + +Usage example: + * Generate csv files which is used for email communication generation. + $ ./generate-data-for-communication.py + --source-openrc ~/c/prod-einfra_cz_migrator.sh.inc + --destination-openrc ~/c/g2-prod-brno-einfra_cz_migrator.sh.inc + --project-name meta-cloud-new-openstack + --signature John Doe + --expiration 1.1.2025 +""" + +import argparse +import logging +import sys +import csv +import yaml + +import lib +import olib +from datetime import date + +def main(args): + """ main project migration loop """ + # connect to source cloud + source_migrator_openrc = lib.get_openrc(args.source_openrc) + source_migrator_conn = lib.get_ostack_connection(source_migrator_openrc) + args.logger.info("A.01 Source OpenStack cloud connected as migrator user") + + # connect to destination cloud + destination_migrator_openrc = lib.get_openrc(args.destination_openrc) + destination_migrator_conn = lib.get_ostack_connection(destination_migrator_openrc) + args.logger.info("A.02 Destination OpenStack cloud connected as migrator user") + + # check project exists in source and destination + source_project_name, destination_project_name = lib.get_ostack_project_names(args.project_name) + source_project = lib.get_ostack_project(source_migrator_conn, source_project_name) + lib.log_or_assert(args, f"B.01 Source OpenStack cloud project (name:{source_project_name}) exists", source_project) + + destination_project = lib.get_ostack_project(destination_migrator_conn, destination_project_name) + if destination_project: + lib.log_or_assert(args, f"B.02 Destination OpenStack cloud project (name:{destination_project_name}) exists", destination_project) + else: + args.logger.warning(f"B.02 Destination OpenStack cloud project (name:{destination_project_name}) does not exist yet") + + # check user context switching & quotas + source_project_conn = lib.get_ostack_connection(source_migrator_openrc | {'OS_PROJECT_NAME': source_project.name}) + destination_project_conn = lib.get_ostack_connection(destination_migrator_openrc | {'OS_PROJECT_NAME': destination_project.name}) + + # get source/destination entities in the project + source_project_servers = lib.get_ostack_project_servers(source_project_conn, source_project) + args.logger.info("C.01 Source OpenStack cloud servers received") + lib.assert_entity_ownership(source_project_servers, source_project) + + destination_project_servers = lib.get_ostack_project_servers(destination_project_conn, destination_project) + args.logger.info("C.02 Destination OpenStack cloud servers received") + lib.assert_entity_ownership(destination_project_servers, destination_project) + + # prepare project data + today = date.today() + vm_count = len(destination_project_servers) + signature = args.signature + project_expiration = args.expiration + + project_data = [ + { + "project_name": source_project_name, + "migration_date": today, + "project_expiration": project_expiration, + "vm_count": vm_count, + "signature": signature, + "servers": "servers.csv" + } + ] + args.logger.info("D.01 Basic information about migrated project gathered") + + # prepare server data + servers_data = [] + for server in destination_project_servers: + server_info = { + "g1_name" : "", + "g1_id" : "", + "g1_fip" : "", + "g2_name" : server.name, + "g2_id" : server.id, + "g2_fip" : get_fip(server.addresses.items()) + } + + for source_server in source_project_servers: + dest_server_name = server.name.replace('migrated-','') + if source_server.name == dest_server_name: + server_info['g1_name'] = source_server.name + server_info['g1_id'] = source_server.id + server_info['g1_fip'] = get_fip(source_server.addresses.items()) + break + servers_data.append(server_info) + args.logger.info("D.02 Information about migrated servers gathered") + + #generate csv files which is lately send to kb job + data_fieldnames = ["project_name", "migration_date", "project_expiration", "vm_count", "signature", "servers"] + write_csv("data.csv", data_fieldnames, project_data) + args.logger.info("E.01 file 'data.csv' containing project data created.") + + servers_fieldnames = ["g1_name", "g1_id", "g1_fip", "g2_name", "g2_id", "g2_fip"] + write_csv("servers.csv", servers_fieldnames, servers_data) + args.logger.info("E.02 file 'servers.csv' containing migrated servers data created.") + +# Function to write data to a CSV file +def write_csv(file_name, fieldnames, data): + with open(file_name, mode='w', newline='') as file: + writer = csv.DictWriter(file, fieldnames=fieldnames, delimiter=';') + writer.writeheader() + writer.writerows(data) + +def get_fip(server_addresses_items): + for network_name,ip_info_list in server_addresses_items: + for ip_info in ip_info_list: + addr = ip_info.get('addr') + ip_type = ip_info.get('OS-EXT-IPS:type') + if ip_type == 'floating': + return addr + +# main() call (argument parsing) +# --------------------------------------------------------------------------- +if __name__ == "__main__": + AP = argparse.ArgumentParser(epilog=globals().get('__doc__'), + formatter_class=argparse.RawDescriptionHelpFormatter) + AP.add_argument('--source-openrc', default=None, type=argparse.FileType('r'), + required=True, help='Source cloud authentication (OpenRC file)') + AP.add_argument('--destination-openrc', default=None, type=argparse.FileType('r'), + required=True, help='Destination cloud authentication (OpenRC file)') + AP.add_argument('--project-name', default=None, required=True, + help='OpenStack project name (identical name in both clouds required)') + AP.add_argument('--signature', default=None, required=True, + help='Signature of person who will be sending the mail.') + AP.add_argument('--expiration', default=None, required=True, + help='Date of expiration of project.') + + AP.add_argument('--exception-trace-file', default="project-migrator.dump", + required=False, + help='Exception / assert dump state file') + AP.add_argument('--log-level', default="INFO", required=False, + choices=[i_lvl for i_lvl in dir(logging) if i_lvl.isupper() and i_lvl.isalpha()], + help='Executio log level (python logging)') + + ARGS = AP.parse_args() + ARGS.logger = logging.getLogger("project-migrator") + logging.basicConfig(level=getattr(logging, ARGS.log_level), + format='%(asctime)s %(name)s %(levelname)s %(message)s') + + sys.exit(main(ARGS)) + \ No newline at end of file