diff --git a/Pipfile b/Pipfile index 7beecd4078364cbcb31e2e76fcff6759827bd198..24478289a7e866c74f37b6138347b36b1514fc59 100644 --- a/Pipfile +++ b/Pipfile @@ -10,8 +10,9 @@ verify_ssl = true [packages] PyYaml = "*" -kypo-python-commons = "==0.1.2" +kypo-python-commons = "==0.1.*" kypo-openstack-lib = { index = "kypo", version = "==0.38.*" } +jinja2 = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 1a51f359adf26313e73c919495eca1fc17cd2f77..2e64d9b0684c531be1a8101cbeab45a8c8d627b2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f00b68a8a481f7758262ac538d9213f21bf540480dd4b1700348b10f71cb5b85" + "sha256": "257adc03759b055e5296aadf24cc950d293fbfdd684332a48a723dabc03b762d" }, "pipfile-spec": 6, "requires": { @@ -209,7 +209,7 @@ "sha256:539835f51a74a69f41b848a9645dbdc35b4f20a3b601e2d9a7e22947b15ff119", "sha256:640bed4bb501cbd17194b3cace1dc2126f5b619cf068a726b98192a0fde74ae9" ], - "markers": "python_version >= '3.7'", + "index": "pypi", "version": "==3.1.1" }, "jmespath": { @@ -252,17 +252,17 @@ }, "kypo-openstack-lib": { "hashes": [ - "sha256:9fb881b2ad685cc824eae315dc8c21bdf62e6b88194eff9080a747394f8d22ef" + "sha256:10924c93160202cec45bb0b6f687d987410d7eb3fa9d82930711746fed7cfab6" ], "index": "kypo", - "version": "==0.38.0" + "version": "==0.38.1" }, "kypo-python-commons": { "hashes": [ - "sha256:1818cd2f7433f5eb358cac402cdb0d096e43419e7f13fce0c4b5400d9f3be010" + "sha256:43917c4e08f37929cf1adc4eaa25d5c22189231a4fceb8b4c5cf9723a788ee14" ], "index": "pypi", - "version": "==0.1.2" + "version": "==0.1.3" }, "kypo-topology-definition": { "hashes": [ @@ -763,11 +763,11 @@ }, "setuptools": { "hashes": [ - "sha256:8f4813dd6a4d6cc17bde85fb2e635fe19763f96efbb0ddf5575562e5ee0bc47a", - "sha256:c3d4e2ab578fbf83775755cd76dae73627915a22832cf4ea5de895978767833b" + "sha256:425ec0e0014c5bcc1104dd1099de6c8f0584854fc9a4f512575f5ed5ee399fb9", + "sha256:6d59c30ce22dd583b42cacf51eebe4c6ea72febaa648aa8b30e5015d23a191fe" ], "markers": "python_version >= '3.7'", - "version": "==61.2.0" + "version": "==61.3.0" }, "simplejson": { "hashes": [ diff --git a/kypo/terraform_driver/templates/terraform_backend.j2 b/kypo/terraform_driver/templates/terraform_backend.j2 new file mode 100644 index 0000000000000000000000000000000000000000..fa70362cff5ed6d64d2effa81d5ad0bc17b7745f --- /dev/null +++ b/kypo/terraform_driver/templates/terraform_backend.j2 @@ -0,0 +1,9 @@ +terraform { + backend "{{ tf_backend }}" { + {%- if tf_backend == "local" %} + path = "{{ tf_state_file_location }}" + {%- else %} + conn_str = "{{ tf_state_file_location }}" + {%- endif %} + } +} diff --git a/kypo/terraform_driver/terraform_backend.py b/kypo/terraform_driver/terraform_backend.py new file mode 100644 index 0000000000000000000000000000000000000000..5d9e2e30d3551e22f4db7af28f0c4e5fce110b5f --- /dev/null +++ b/kypo/terraform_driver/terraform_backend.py @@ -0,0 +1,48 @@ +import os + +from jinja2 import Environment, FileSystemLoader +from kypo.cloud_commons import KypoException + +from kypo.terraform_driver.terraform_client_elements import KypoTerraformBackendType + +TERRAFORM_STATE_FILE_NAME = 'terraform.tfstate' +TEMPLATES_DIR_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'templates') +TERRAFORM_BACKEND_FILE_NAME = 'terraform_backend.j2' + + +class KypoTerraformBackend: + + def __init__(self, backend_type: KypoTerraformBackendType, db_configuration=None): + self.backend_type = backend_type + self.db_configuration = db_configuration + self.template_environment = Environment(loader=(FileSystemLoader(TEMPLATES_DIR_PATH))) + self.template = self._create_terraform_backend_template() + + def _get_state_file_location(self) -> str: + """ + Get state file location for Terraform backend configuration. + :return: State file locaiton + """ + if self.backend_type == KypoTerraformBackendType.LOCAL: + return TERRAFORM_STATE_FILE_NAME + + if self.db_configuration is None: + raise KypoException(f'Cannot use backend "{self.backend_type.value()}" without' + f' specifying database configuration.') + + try: + return 'postgres://{0[user]}:{0[password]}@{0[host]}/{0[name]}'\ + .format(self.db_configuration) + except KeyError as exc: + raise KypoException(f'Database configuration is incomplete. Error: "{exc}"') + + def _create_terraform_backend_template(self) -> str: + """ + Create Terraform backend configuration + :return: Terraform backend configuration + """ + template = self.template_environment.get_template(TERRAFORM_BACKEND_FILE_NAME) + return template.render( + tf_backend=self.backend_type, + tf_state_file_location=self._get_state_file_location(), + ) diff --git a/kypo/terraform_driver/terraform_client.py b/kypo/terraform_driver/terraform_client.py index 3a4433a025f9c5a69ced414513ebe0d4dedff46b..a1ed689234b4f645fec09676a5b80b7d6c125af3 100644 --- a/kypo/terraform_driver/terraform_client.py +++ b/kypo/terraform_driver/terraform_client.py @@ -5,10 +5,11 @@ from kypo.cloud_commons import KypoCloudClientBase, TopologyInstance, Transforma Image, Limits, QuotaSet, HardwareUsage # Available cloud clients from kypo.openstack_driver import KypoOpenStackClient - from kypo.topology_definition.models import TopologyDefinition -from kypo.terraform_driver.terraform_client_elements import TerraformInstance +from kypo.terraform_driver.terraform_backend import KypoTerraformBackend +from kypo.terraform_driver.terraform_client_elements import TerraformInstance, \ + KypoTerraformBackendType from kypo.terraform_driver.terraform_client_manager import KypoTerraformClientManager @@ -22,10 +23,14 @@ class KypoTerraformClient: """ def __init__(self, cloud_client: AvailableCloudLibraries, trc: TransformationConfiguration, - stacks_dir: str = None, template_file_name: str = None, *args, **kwargs): + stacks_dir: str = None, template_file_name: str = None, + backend_type: KypoTerraformBackendType = KypoTerraformBackendType('local'), + db_configuration=None, *args, **kwargs): self.cloud_client: KypoCloudClientBase = cloud_client.value(trc=trc, *args, **kwargs) + terraform_backend = KypoTerraformBackend(backend_type=backend_type, + db_configuration=db_configuration) self.client_manager = KypoTerraformClientManager(stacks_dir, self.cloud_client, trc, - template_file_name) + template_file_name, terraform_backend) self.trc = trc def get_process_output(self, process): diff --git a/kypo/terraform_driver/terraform_client_elements.py b/kypo/terraform_driver/terraform_client_elements.py index 682f7e120dc9438fa141079dbd87d33eddc430e5..d870922c14551e48ae093e73c4eb5823c8f65d5b 100644 --- a/kypo/terraform_driver/terraform_client_elements.py +++ b/kypo/terraform_driver/terraform_client_elements.py @@ -1,7 +1,14 @@ -from typing import List, Union, Dict +from enum import Enum +from typing import Union, Dict + from kypo.cloud_commons.cloud_client_elements import Image +class KypoTerraformBackendType(Enum): + LOCAL = 'local' + POSTGRES = 'pg' + + class TerraformInstance: """ Used to represent terraform stack instance diff --git a/kypo/terraform_driver/terraform_client_manager.py b/kypo/terraform_driver/terraform_client_manager.py index 47c8a029ccc78a08ab62bc7a93b4ae3af9341ca2..e5a5b46a4792bdf968836ea33d8efab9fdf9ac92 100644 --- a/kypo/terraform_driver/terraform_client_manager.py +++ b/kypo/terraform_driver/terraform_client_manager.py @@ -7,10 +7,13 @@ from typing import List from kypo.cloud_commons import StackNotFound, KypoException, Image, TopologyInstance from kypo.terraform_driver.terraform_client_elements import TerraformInstance +from kypo.terraform_driver.terraform_backend import KypoTerraformBackend, TERRAFORM_STATE_FILE_NAME +from kypo.terraform_driver.terraform_exceptions import TerraformInitFailed, TerraformWorkspaceFailed STACKS_DIR = '/var/tmp/kypo/terraform-stacks/' TEMPLATE_FILE_NAME = 'deploy.tf' -TERRAFORM_STATE_FILE_NAME = 'terraform.tfstate' +TERRAFORM_BACKEND_FILE_NAME = 'backend.tf' +TERRAFORM_PROVIDER_FILE_NAME = 'provider.tf' class KypoTerraformClientManager: @@ -18,12 +21,53 @@ class KypoTerraformClientManager: Manager class for KypoTerraformClient """ - def __init__(self, stacks_dir, cloud_client, trc, template_file_name): + def __init__(self, stacks_dir, cloud_client, trc, template_file_name, + terraform_backend: KypoTerraformBackend): self.cloud_client = cloud_client self.stacks_dir = stacks_dir if stacks_dir else STACKS_DIR self.template_file_name = template_file_name if template_file_name else TEMPLATE_FILE_NAME self.trc = trc self.create_directories(self.stacks_dir) + self.terraform_backend = terraform_backend + + def _create_terraform_backend_file(self, stack_dir: str) -> None: + """ + Create backend.tf file containing configuration for Terraform backend. + + :param stack_dir: The path to the stack directory + :return: None + """ + template = self.terraform_backend.template + + self.create_file(os.path.join(stack_dir, TERRAFORM_BACKEND_FILE_NAME), template) + + def _create_terraform_provider(self, stack_dir) -> None: + """ + Create file with Terraform provider configuration. + :param stack_dir: The path to the stack directory + :return: None + """ + provider = self.cloud_client.get_terraform_provider() + + self.create_file(os.path.join(stack_dir, TERRAFORM_PROVIDER_FILE_NAME), provider) + + def _initialize_stack_dir(self, stack_name: str, terraform_template: str = None) -> None: + """ + + :param stack_name: The name of Terraform stack. + :param terraform_template: Terraform template specifying resources of the stack. + :return: None + :raise KypoException: If should_raise is True and Terraform command fails. + """ + stack_dir = self.get_stack_dir(stack_name) + self.create_directories(stack_dir) + self._create_terraform_backend_file(stack_dir) + self._create_terraform_provider(stack_dir) + + if terraform_template: + self.create_file(os.path.join(stack_dir, self.template_file_name), terraform_template) + + self.init_terraform(stack_dir, stack_name) @staticmethod def create_directories(dir_path: str) -> None: @@ -102,16 +146,29 @@ class KypoTerraformClientManager: """ return os.path.join(self.stacks_dir, stack_name) - def init_terraform(self, stack_dir: str) -> None: + def init_terraform(self, stack_dir: str, stack_name: str) -> None: """ Initialize Terraform properties in stack directory. :param stack_dir: Path to the stack directory + :param stack_name: The name of Terraform stack :return: None - :raise KypoException: Terraform initialization failed + :raise TerraformInitFailed: The 'terraform init' command fails. + :raise TerraformWorkspaceFailed: Could not create new workspace. """ - process = subprocess.Popen(['terraform', 'init'], cwd=stack_dir, stdout=subprocess.PIPE) - self.wait_for_process(process) + try: + process = subprocess.Popen(['terraform', 'init'], cwd=stack_dir, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.wait_for_process(process) + except KypoException as exc: + raise TerraformInitFailed(exc) + + try: + process = subprocess.Popen(['terraform', 'workspace', 'new', stack_name], cwd=stack_dir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.wait_for_process(process) + except KypoException as exc: + raise TerraformWorkspaceFailed(exc) def create_terraform_template(self, topology_instance: TopologyInstance, *args, **kwargs)\ -> str: @@ -145,9 +202,7 @@ class KypoTerraformClientManager: resource_prefix=stack_name, *args, **kwargs) stack_dir = self.get_stack_dir(stack_name) - self.create_directories(stack_dir) - self.create_file(os.path.join(stack_dir, self.template_file_name), terraform_template) - self.init_terraform(stack_dir) # TODO: Can fail + self._initialize_stack_dir(stack_name, terraform_template) if dry_run: return subprocess.Popen(['terraform', 'plan'], cwd=stack_dir, stdout=subprocess.PIPE, @@ -166,6 +221,13 @@ class KypoTerraformClientManager: :raise KypoException: Stack deletion has failed """ stack_dir = self.get_stack_dir(stack_name) + try: + self._initialize_stack_dir(stack_name) + except TerraformInitFailed: + return None + except TerraformWorkspaceFailed: + pass + return subprocess.Popen(['terraform', 'destroy', '-auto-approve', '-no-color'], cwd=stack_dir, stdout=subprocess.PIPE, text=True) diff --git a/kypo/terraform_driver/terraform_exceptions.py b/kypo/terraform_driver/terraform_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..dec134f300c65c293cd8e38d79e972687be49c1c --- /dev/null +++ b/kypo/terraform_driver/terraform_exceptions.py @@ -0,0 +1,19 @@ +""" +Module containing KYPO Terraform exceptions. +""" + +from kypo.cloud_commons import KypoException + + +class TerraformInitFailed(KypoException): + """ + This exception is raised if 'terraform init' command fails. + """ + pass + + +class TerraformWorkspaceFailed(KypoException): + """ + This exception is raised if `terraform workspace` command fails. + """ + pass diff --git a/setup.py b/setup.py index b2ba232f9aec0a2e818be952882ab07c77cee4c0..47b3ccf7ef1bc2edd4f96509dafeffc35b290a90 100644 --- a/setup.py +++ b/setup.py @@ -17,8 +17,9 @@ setup( long_description=read('README.md'), packages=find_namespace_packages(include=['kypo.*'], exclude=['tests']), install_requires=[ - 'kypo-python-commons==0.1.2', + 'kypo-python-commons==0.1.*', 'kypo-openstack-lib==0.38.*', + 'Jinja2', ], python_requires='>=3', zip_safe=False