Merge branch '1-save-terraform-state-as-remote-state-to-database' into 'master'

Resolve "Save terraform state as remote state to database"

Closes #1

See merge request !1

Pipfile

@@ -10,8 +10,9 @@ verify_ssl = true
 
 [packages]
 PyYaml = "*"
-kypo-python-commons = "==0.1.2"
+kypo-python-commons = "==0.1.*"
 kypo-openstack-lib = { index = "kypo", version = "==0.38.*" }
+jinja2 = "*"
 
 [dev-packages]
 

Pipfile.lock

 {
     "_meta": {
         "hash": {
-            "sha256": "f00b68a8a481f7758262ac538d9213f21bf540480dd4b1700348b10f71cb5b85"
+            "sha256": "257adc03759b055e5296aadf24cc950d293fbfdd684332a48a723dabc03b762d"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -209,7 +209,7 @@
                 "sha256:539835f51a74a69f41b848a9645dbdc35b4f20a3b601e2d9a7e22947b15ff119",
                 "sha256:640bed4bb501cbd17194b3cace1dc2126f5b619cf068a726b98192a0fde74ae9"
             ],
-            "markers": "python_version >= '3.7'",
+            "index": "pypi",
             "version": "==3.1.1"
         },
         "jmespath": {
@@ -252,17 +252,17 @@
         },
         "kypo-openstack-lib": {
             "hashes": [
-                "sha256:9fb881b2ad685cc824eae315dc8c21bdf62e6b88194eff9080a747394f8d22ef"
+                "sha256:10924c93160202cec45bb0b6f687d987410d7eb3fa9d82930711746fed7cfab6"
             ],
             "index": "kypo",
-            "version": "==0.38.0"
+            "version": "==0.38.1"
         },
         "kypo-python-commons": {
             "hashes": [
-                "sha256:1818cd2f7433f5eb358cac402cdb0d096e43419e7f13fce0c4b5400d9f3be010"
+                "sha256:43917c4e08f37929cf1adc4eaa25d5c22189231a4fceb8b4c5cf9723a788ee14"
             ],
             "index": "pypi",
-            "version": "==0.1.2"
+            "version": "==0.1.3"
         },
         "kypo-topology-definition": {
             "hashes": [
@@ -763,11 +763,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:8f4813dd6a4d6cc17bde85fb2e635fe19763f96efbb0ddf5575562e5ee0bc47a",
-                "sha256:c3d4e2ab578fbf83775755cd76dae73627915a22832cf4ea5de895978767833b"
+                "sha256:425ec0e0014c5bcc1104dd1099de6c8f0584854fc9a4f512575f5ed5ee399fb9",
+                "sha256:6d59c30ce22dd583b42cacf51eebe4c6ea72febaa648aa8b30e5015d23a191fe"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==61.2.0"
+            "version": "==61.3.0"
         },
         "simplejson": {
             "hashes": [

kypo/terraform_driver/templates/terraform_backend.j2

+terraform {
+  backend "{{ tf_backend }}" {
+    {%- if tf_backend == "local" %}
+    path = "{{ tf_state_file_location }}"
+    {%- else %}
+    conn_str = "{{ tf_state_file_location }}"
+    {%- endif %}
+  }
+}

kypo/terraform_driver/terraform_backend.py

+import os
+
+from jinja2 import Environment, FileSystemLoader
+from kypo.cloud_commons import KypoException
+
+from kypo.terraform_driver.terraform_client_elements import KypoTerraformBackendType
+
+TERRAFORM_STATE_FILE_NAME = 'terraform.tfstate'
+TEMPLATES_DIR_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'templates')
+TERRAFORM_BACKEND_FILE_NAME = 'terraform_backend.j2'
+
+
+class KypoTerraformBackend:
+
+    def __init__(self, backend_type: KypoTerraformBackendType, db_configuration=None):
+        self.backend_type = backend_type
+        self.db_configuration = db_configuration
+        self.template_environment = Environment(loader=(FileSystemLoader(TEMPLATES_DIR_PATH)))
+        self.template = self._create_terraform_backend_template()
+
+    def _get_state_file_location(self) -> str:
+        """
+        Get state file location for Terraform backend configuration.
+        :return: State file locaiton
+        """
+        if self.backend_type == KypoTerraformBackendType.LOCAL:
+            return TERRAFORM_STATE_FILE_NAME
+
+        if self.db_configuration is None:
+            raise KypoException(f'Cannot use backend "{self.backend_type.value()}" without'
+                                f' specifying database configuration.')
+
+        try:
+            return 'postgres://{0[user]}:{0[password]}@{0[host]}/{0[name]}'\
+                .format(self.db_configuration)
+        except KeyError as exc:
+            raise KypoException(f'Database configuration is incomplete. Error: "{exc}"')
+
+    def _create_terraform_backend_template(self) -> str:
+        """
+        Create Terraform backend configuration
+        :return: Terraform backend configuration
+        """
+        template = self.template_environment.get_template(TERRAFORM_BACKEND_FILE_NAME)
+        return template.render(
+            tf_backend=self.backend_type,
+            tf_state_file_location=self._get_state_file_location(),
+        )

kypo/terraform_driver/terraform_client.py

@@ -5,10 +5,11 @@ from kypo.cloud_commons import KypoCloudClientBase, TopologyInstance, Transforma
     Image, Limits, QuotaSet, HardwareUsage
 # Available cloud clients
 from kypo.openstack_driver import KypoOpenStackClient
-
 from kypo.topology_definition.models import TopologyDefinition
 
-from kypo.terraform_driver.terraform_client_elements import TerraformInstance
+from kypo.terraform_driver.terraform_backend import KypoTerraformBackend
+from kypo.terraform_driver.terraform_client_elements import TerraformInstance, \
+    KypoTerraformBackendType
 from kypo.terraform_driver.terraform_client_manager import KypoTerraformClientManager
 
 
@@ -22,10 +23,14 @@ class KypoTerraformClient:
     """
 
     def __init__(self, cloud_client: AvailableCloudLibraries, trc: TransformationConfiguration,
-                 stacks_dir: str = None, template_file_name: str = None, *args, **kwargs):
+                 stacks_dir: str = None, template_file_name: str = None,
+                 backend_type: KypoTerraformBackendType = KypoTerraformBackendType('local'),
+                 db_configuration=None, *args, **kwargs):
         self.cloud_client: KypoCloudClientBase = cloud_client.value(trc=trc, *args, **kwargs)
+        terraform_backend = KypoTerraformBackend(backend_type=backend_type,
+                                                 db_configuration=db_configuration)
         self.client_manager = KypoTerraformClientManager(stacks_dir, self.cloud_client, trc,
-                                                         template_file_name)
+                                                         template_file_name, terraform_backend)
         self.trc = trc
 
     def get_process_output(self, process):

kypo/terraform_driver/terraform_client_elements.py

-from typing import List, Union, Dict
+from enum import Enum
+from typing import Union, Dict
+
 from kypo.cloud_commons.cloud_client_elements import Image
 
 
+class KypoTerraformBackendType(Enum):
+    LOCAL = 'local'
+    POSTGRES = 'pg'
+
+
 class TerraformInstance:
     """
     Used to represent terraform stack instance

kypo/terraform_driver/terraform_client_manager.py

@@ -7,10 +7,13 @@ from typing import List
 from kypo.cloud_commons import StackNotFound, KypoException, Image, TopologyInstance
 
 from kypo.terraform_driver.terraform_client_elements import TerraformInstance
+from kypo.terraform_driver.terraform_backend import KypoTerraformBackend, TERRAFORM_STATE_FILE_NAME
+from kypo.terraform_driver.terraform_exceptions import TerraformInitFailed, TerraformWorkspaceFailed
 
 STACKS_DIR = '/var/tmp/kypo/terraform-stacks/'
 TEMPLATE_FILE_NAME = 'deploy.tf'
-TERRAFORM_STATE_FILE_NAME = 'terraform.tfstate'
+TERRAFORM_BACKEND_FILE_NAME = 'backend.tf'
+TERRAFORM_PROVIDER_FILE_NAME = 'provider.tf'
 
 
 class KypoTerraformClientManager:
@@ -18,12 +21,53 @@ class KypoTerraformClientManager:
     Manager class for KypoTerraformClient
     """
 
-    def __init__(self, stacks_dir, cloud_client, trc, template_file_name):
+    def __init__(self, stacks_dir, cloud_client, trc, template_file_name,
+                 terraform_backend: KypoTerraformBackend):
         self.cloud_client = cloud_client
         self.stacks_dir = stacks_dir if stacks_dir else STACKS_DIR
         self.template_file_name = template_file_name if template_file_name else TEMPLATE_FILE_NAME
         self.trc = trc
         self.create_directories(self.stacks_dir)
+        self.terraform_backend = terraform_backend
+
+    def _create_terraform_backend_file(self, stack_dir: str) -> None:
+        """
+        Create backend.tf file containing configuration for Terraform backend.
+
+        :param stack_dir: The path to the stack directory
+        :return: None
+        """
+        template = self.terraform_backend.template
+
+        self.create_file(os.path.join(stack_dir, TERRAFORM_BACKEND_FILE_NAME), template)
+
+    def _create_terraform_provider(self, stack_dir) -> None:
+        """
+        Create file with Terraform provider configuration.
+        :param stack_dir: The path to the stack directory
+        :return: None
+        """
+        provider = self.cloud_client.get_terraform_provider()
+
+        self.create_file(os.path.join(stack_dir, TERRAFORM_PROVIDER_FILE_NAME), provider)
+
+    def _initialize_stack_dir(self, stack_name: str, terraform_template: str = None) -> None:
+        """
+
+        :param stack_name: The name of Terraform stack.
+        :param terraform_template: Terraform template specifying resources of the stack.
+        :return: None
+        :raise KypoException: If should_raise is True and Terraform command fails.
+        """
+        stack_dir = self.get_stack_dir(stack_name)
+        self.create_directories(stack_dir)
+        self._create_terraform_backend_file(stack_dir)
+        self._create_terraform_provider(stack_dir)
+
+        if terraform_template:
+            self.create_file(os.path.join(stack_dir, self.template_file_name), terraform_template)
+
+        self.init_terraform(stack_dir, stack_name)
 
     @staticmethod
     def create_directories(dir_path: str) -> None:
@@ -102,16 +146,29 @@ class KypoTerraformClientManager:
         """
         return os.path.join(self.stacks_dir, stack_name)
 
-    def init_terraform(self, stack_dir: str) -> None:
+    def init_terraform(self, stack_dir: str, stack_name: str) -> None:
         """
         Initialize Terraform properties in stack directory.
 
         :param stack_dir: Path to the stack directory
+        :param stack_name: The name of Terraform stack
         :return: None
-        :raise KypoException: Terraform initialization failed
+        :raise TerraformInitFailed: The 'terraform init' command fails.
+        :raise TerraformWorkspaceFailed: Could not create new workspace.
         """
-        process = subprocess.Popen(['terraform', 'init'], cwd=stack_dir, stdout=subprocess.PIPE)
+        try:
+            process = subprocess.Popen(['terraform', 'init'], cwd=stack_dir, stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE)
             self.wait_for_process(process)
+        except KypoException as exc:
+            raise TerraformInitFailed(exc)
+
+        try:
+            process = subprocess.Popen(['terraform', 'workspace', 'new', stack_name], cwd=stack_dir,
+                                       stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            self.wait_for_process(process)
+        except KypoException as exc:
+            raise TerraformWorkspaceFailed(exc)
 
     def create_terraform_template(self, topology_instance: TopologyInstance, *args, **kwargs)\
             -> str:
@@ -145,9 +202,7 @@ class KypoTerraformClientManager:
                                                             resource_prefix=stack_name, *args,
                                                             **kwargs)
         stack_dir = self.get_stack_dir(stack_name)
-        self.create_directories(stack_dir)
-        self.create_file(os.path.join(stack_dir, self.template_file_name), terraform_template)
-        self.init_terraform(stack_dir)  # TODO: Can fail
+        self._initialize_stack_dir(stack_name, terraform_template)
 
         if dry_run:
             return subprocess.Popen(['terraform', 'plan'], cwd=stack_dir, stdout=subprocess.PIPE,
@@ -166,6 +221,13 @@ class KypoTerraformClientManager:
         :raise KypoException: Stack deletion has failed
         """
         stack_dir = self.get_stack_dir(stack_name)
+        try:
+            self._initialize_stack_dir(stack_name)
+        except TerraformInitFailed:
+            return None
+        except TerraformWorkspaceFailed:
+            pass
+
         return subprocess.Popen(['terraform', 'destroy', '-auto-approve', '-no-color'],
                                 cwd=stack_dir, stdout=subprocess.PIPE, text=True)
 

kypo/terraform_driver/terraform_exceptions.py

+"""
+Module containing KYPO Terraform exceptions.
+"""
+
+from kypo.cloud_commons import KypoException
+
+
+class TerraformInitFailed(KypoException):
+    """
+    This exception is raised if 'terraform init' command fails.
+    """
+    pass
+
+
+class TerraformWorkspaceFailed(KypoException):
+    """
+    This exception is raised if `terraform workspace` command fails.
+    """
+    pass

setup.py

@@ -17,8 +17,9 @@ setup(
     long_description=read('README.md'),
     packages=find_namespace_packages(include=['kypo.*'], exclude=['tests']),
     install_requires=[
-        'kypo-python-commons==0.1.2',
+        'kypo-python-commons==0.1.*',
         'kypo-openstack-lib==0.38.*',
+        'Jinja2',
     ],
     python_requires='>=3',
     zip_safe=False