refactor: parametrize the terraform module with tfvars files

clouds/common/general/terraform/terraform-demo.sh

+#!/usr/bin/env bash
+# e-INFRA CZ G2 openstack terraform demo - personal/group project
+# Usage: terraform-demo.sh [ostack-entities-prefix/profile-name]
+
+
+SCRIPT_DIR=$(dirname $(readlink -f $0))
+#############################################################################
+# variables
+#############################################################################
+ENTITIES_PREFIX="${1:-"${USER}_$(hostname)"}"
+
+#############################################################################
+# functions
+#############################################################################
+source ${SCRIPT_DIR}/../../../../common/lib.sh.inc
+
+#############################################################################
+# main steps
+#############################################################################
+log_section "Using commandline tools:"
+
+./terraform.sh validate-tools
+
+log_section "Using commandline tools:"
+./terraform.sh detect-cloud
+./terraform.sh detect-project
+./terraform.sh init
+./terraform.sh validate
+./terraform.sh plan --out plan
+./terraform.sh apply plan
+./terraform.sh destroy
+

clouds/common/general/terraform/terraform.sh

+#!/usr/bin/env bash
+# terraform wrapper for automatic personal/group project reconfiguration
+# terraform.sh <arguments>
+#
+# <arguments> are:
+# * detect-cloud
+# * detect-project
+# * validate-tools
+# * or any valid terraform arguments
+
+# functions
+SCRIPT_DIR=$(dirname $(readlink -f $0))
+source ${SCRIPT_DIR}/../../../../common/lib.sh.inc
+
+
+if [[ "$1" =~ (detect-(cloud|project)|validate-tools) ]]; then
+    if [ "$1" == "detect-cloud" ]; then
+        log "Using OpenStack cloud:"
+        openstack version show -fcsv | grep identity
+    elif [ "$1" == "detect-project" ]; then
+        project_type=group
+        if prj_name=$(is_personal_project); then
+            project_type=personal
+        fi
+        log "Using OpenStack ${project_type} project named: ${prj_name}"
+    elif [ "$1" == "validate-tools" ]; then
+        log "Using commandline tools:"
+        report_tools "terraform version"
+    fi
+    exit $?
+else
+    project_type=group
+    if prj_name=$(is_personal_project); then
+        project_type=personal
+        if [ "$1" == "plan" ]; then
+            terraform "$@" --var-file=personal-projects.tfvars
+        else
+            terraform "$@"
+        fi
+    else
+        terraform "$@"
+    fi
+fi
+
+
+
+
+

clouds/common/general/terraform/variables.tf

+variable "router_creation_enable" {
+  description = "Create dedicated router instance. true/false ~ create new / reuse existing personal router"
+  default     = true
+}
+
+variable "internal_network_creation_enable" {
+  description = "Create dedicated internal network. true/false ~ create new / reuse existing personal network"
+  default     = true
+}
+
+variable "internal_network_name" {
+  description = "Internal network name. Either dedicated new network or existing personal network name"
+  default     = "<var.infra_name>_network"
+}
+
+variable "internal_subnet_creation_enable" {
+  description = "Create dedicated subnet instance. true/false ~ create new / reuse existing personal subnet"
+  default     = true
+}
+
+variable "internal_subnet_name" {
+  description = "Internal network subnet name. Either dedicated new subnet or existing personal subnet name"
+  default     = "<var.infra_name>_subnet"
+}

clouds/g1-to-g2-migration/README.md

+# e-INFRA CZ G1 to G2 cloud migration resources

clouds/g1-to-g2-migration/general/terraform/README.md

+# Terraform demonstration
+
+This Terraform module is able to demonstrate creation of two most used cloud infrastructure patterns:
+
+## Two tier infrastructure: public bastion and private VM farm
+
+Infrastructure consist of:
+ - public facing small bastion VM (sometimes called as jump VM)
+ - private VM farm
+
+![two-tier-infra.png](/clouds/common/pictures/two-tier-infra.png)
+
+## Single tier infrastructure i.e. public facing VM farm
+
+Infrastructure consist of:
+ - public facing VM farm
+
+![single-tier-infra.png](/clouds/common/pictures/single-tier-infra.png)
+
+
+## Cloud VM configuration during system boot using cloud-init
+Terraform demonstrates [how to configure VM servers on creation with cloud-init](modules/common/nodes-cloudinit.txt):
+ - Add ssh keys, disable SSH password auth
+ - Create partition and filesystemand mount extra data from extra volume
+
+## Handling different project restrictions (quotas, shared networks, ...)
+
+e-INFRA.CZ OpenStack cloud distinguishes between two project types: personal and group
+([more the topic](https://docs.e-infra.cz/compute/openstack/technical-reference/brno-site/get-access/#personal-project)).
+Terraform demo code with additional variable file [`personal-projects.tfvars`](./personal-projects.tfvars) shows how to support both project types i.e. how to parametrize infrastructure definition with minimal code duplications.
+
+There is thin terraform wrapper [`terraform.sh`](./terraform.sh) abstracting the fact which project type you are on. It is not necessary to use the [`terraform.sh`](./terraform.sh) wrapper when you keep in mind that additional terraform variables file [`personal-projects.tfvars`](./personal-projects.tfvars) has to be passed at plan step when you are on personal project.
+
+## Using the terraform demo
+
+1. Clone the repository.
+1. Load you OpenStack application credentials to environment variables `source project_openrc.sh.inc`
+1. Override any infrastructure variables in [main.tf](main.tf) file if needed. Full set of variables can be found in [modules/common/variables.tf](modules/common/variables.tf).
+1. In the [terraform root directory](/clouds/g2/ostrava/general/terraform) run following commands to initiate and validate environment
+   * `./terraform.sh validate-tools`
+   * `./terraform.sh detect-cloud`
+   * `./terraform.sh detect-project`
+   * `./terraform.sh init`
+   * `./terraform.sh validate`
+1. In the [same directory](/clouds/g2/ostrava/general/terraform) run commands to deploy cloud infrastructure
+   * `./terraform.sh plan --out plan`
+   * `./terraform.sh apply plan`
+1. Once you need to change the infrastructure, first modify the infrastructure declaration and repeat above steps to deploy changes.
+1. Cloud resources can be deleted with `./terraform.sh destroy`.
+
+> You may use terraform directly without the [`terraform.sh`](./terraform.sh) wrapper, but then keep in your mind that just for personal project type you should pass additional variable file [`personal-projects.tfvars`](./personal-projects.tfvars) at plan step (i.e. `terraform plan --out personal-project.plan --var-file=personal-projects.tfvars`)
+
+Detailed terminal transcripts show how to run terraform demo inside personal project ([with](./terraform-2tier_public_bastion_private_vm_farm-personal-project-wrapper.log) / [without the wrapper](./terraform-2tier_public_bastion_private_vm_farm-personal-project-no-wrapper.log)) as well as under group project([with](./terraform-2tier_public_bastion_private_vm_farm-group-project-wrapper.log) / [without the wrapper](./terraform-2tier_public_bastion_private_vm_farm-group-project-no-wrapper.log)).
+
+
+## Access to the VM nodes
+
+In single tier infrastructure you access directly the individual VM nodes via SSH on public IP addresses.
+Two tier infrastructure requires the access following way:
+1. Establish the connection with bastion
+```sh
+sshuttle -r ubuntu@<bastion-ip>
+```
+1. Connect directly to VM nodes via SSH on private IP addresses:
+```sh
+ssh  ubuntu@<vm-node-ip-from-10.10.10.0/24>
+```

clouds/g1-to-g2-migration/general/terraform/g1-brno-group-projects.tfvars

+# extra variabes for an OpenStack group project

clouds/g1-to-g2-migration/general/terraform/g1-brno-personal-projects.tfvars

+# extra variabes for an OpenStack personal project
+router_creation_enable           = false
+internal_network_creation_enable = false
+internal_network_name            = "internal-ipv4-general-private"
+internal_subnet_creation_enable  = false
+internal_subnet_name             = "internal-ipv4-general-private-172-16-0-0"

clouds/g1-to-g2-migration/general/terraform/g1-prod-brno-main.tf

-terraform {
-  backend "local" {}
-}
-
-module "toplevel" {
-  # two tier infrastructure (2tier_public_bastion_private_vm_farm module):
-  # * single public facing tiny bastion VM
-  # * <nodes_count> private HPC VM farm
-  source                  = "./modules/2tier_public_bastion_private_vm_farm"
-  # single tier infrastructure (1tier_public_vm_farm monule)
-  # * <nodes_count> public HPC VM farm
-  #source                  = "./modules/1tier_public_vm_farm"
-
-  infra_name              = "general-tf-demo"
-
-  nodes_count             = 2
-  nodes_extra_volume_size = 20  # in GB
-
-  #nodes_flavor            = "standard.medium"
-  #nodes_image             = "ubuntu-jammy-x86_64"
-}

clouds/g1-to-g2-migration/general/terraform/g2-brno-group-projects.tfvars

+# extra variabes for an OpenStack group project

clouds/g1-to-g2-migration/general/terraform/g2-brno-personal-projects.tfvars

+# extra variabes for an OpenStack personal project
+router_creation_enable           = false
+internal_network_creation_enable = false
+internal_network_name            = "internal-ipv4-general-private"
+internal_subnet_creation_enable  = false
+internal_subnet_name             = "internal-ipv4-general-private-172-16-0-0"

clouds/g1-to-g2-migration/general/terraform/g2-ostrava-group-projects.tfvars

+# extra variabes for an OpenStack group project

clouds/g1-to-g2-migration/general/terraform/g2-ostrava-personal-projects.tfvars

+# extra variabes for an OpenStack personal project
+router_creation_enable           = false
+internal_network_creation_enable = false
+internal_network_name            = "internal-ipv4-general-private"
+internal_subnet_creation_enable  = false
+internal_subnet_name             = "internal-ipv4-general-private-172-16-0-0"

clouds/g1-to-g2-migration/general/terraform/g2-prod-ostrava-main.tf

-terraform {
-  backend "local" {}
-}
-
-module "toplevel" {
-  # infrastructure type:
-  # -------------------------------------------------------------------------
-  # two tier infrastructure (2tier_public_bastion_private_vm_farm module):
-  # * single public facing tiny bastion VM
-  # * <nodes_count> private HPC VM farm
-  source                  = "./modules/2tier_public_bastion_private_vm_farm"
-  # single tier infrastructure (1tier_public_vm_farm monule)
-  # * <nodes_count> public HPC VM farm
-  #source                  = "./modules/1tier_public_vm_farm"
-
-  infra_name              = "freznicek-general-tf-demo"
-  bastion_flavor          = "e1.small"
-
-  nodes_count             = 1
-  nodes_flavor            = "e1.small"
-  nodes_image             = "ubuntu-jammy-x86_64"
-  public_external_network = "external-ipv4-general-public"
-  nodes_extra_volume_size = 10  # extra volume size in GB
-
-  # OpenStack project type:
-  # -------------------------------------------------------------------------
-  # root variables wired 1:1 to "toplevel" module to be able to toggle between
-  # group and personal project infrastructure
-  router_creation_enable           = var.router_creation_enable
-  internal_network_creation_enable = var.internal_network_creation_enable
-  internal_network_name            = var.internal_network_name
-  internal_subnet_creation_enable  = var.internal_subnet_creation_enable
-  internal_subnet_name             = var.internal_subnet_name
-}

clouds/g1-to-g2-migration/general/terraform/main.tf

+../../../common/general/terraform/main.tf
\ No newline at end of file

clouds/g1-to-g2-migration/general/terraform/terraform-demo.sh

+../../../common/general/terraform/terraform-demo.sh
\ No newline at end of file

clouds/g1-to-g2-migration/general/terraform/terraform.sh

+../../../common/general/terraform/terraform.sh
\ No newline at end of file

clouds/g1-to-g2-migration/general/terraform/variables.tf

+../../../common/general/terraform/variables.tf
\ No newline at end of file