From 70dc20cf1e825975faffdd6009763c4596c87b9a Mon Sep 17 00:00:00 2001
From: Attila Farkas <ati@mail.muni.cz>
Date: Fri, 6 Mar 2020 14:17:01 +0100
Subject: [PATCH] rewrite vagrantfile generation
---
conf/flavors.yml | 40 +++++++++
conf/router_attributes.yml | 3 +
conf/vagrant_mapping.yml | 6 --
conf/virtualbox_mapping.yml | 3 +
create.py | 7 +-
modules/ansible_generator.py | 6 ++
modules/file_generator.py | 52 ++---------
modules/file_manager.py | 39 ++++++++
modules/preprocessing.py | 44 ++++++++-
modules/vagrant_generator.py | 169 ++++++++++++++++++++++++++++++++---
templates/newvagrantfile | 111 +++++++++++++++++++++++
11 files changed, 414 insertions(+), 66 deletions(-)
create mode 100644 conf/flavors.yml
create mode 100644 conf/router_attributes.yml
create mode 100644 conf/virtualbox_mapping.yml
create mode 100644 modules/ansible_generator.py
create mode 100644 modules/file_manager.py
create mode 100644 templates/newvagrantfile
diff --git a/conf/flavors.yml b/conf/flavors.yml
new file mode 100644
index 0000000..55a4ceb
--- /dev/null
+++ b/conf/flavors.yml
@@ -0,0 +1,40 @@
+csirtmu.tiny1x2:
+ cores: 1
+ memory: 2048
+ hd: 20480
+csirtmu.tiny1x4:
+ cores: 1
+ memory: 4096
+ hd: 20480
+csirtmu.small2x4:
+ cores: 2
+ memory: 4096
+ hd: 20480
+csirtmu.small2x8:
+ cores: 2
+ memory: 8192
+ hd: 40960
+csirtmu.medium4x8:
+ cores: 4
+ memory: 8192
+ hd: 40960
+csirtmu.medium4x16:
+ cores: 4
+ memory: 16384
+ hd: 40960
+csirtmu.large8x16:
+ cores: 8
+ memory: 16384
+ hd: 81920
+csirtmu.large8x32:
+ cores: 8
+ memory: 32768
+ hd: 81920
+csirtmu.jumbo16x32:
+ cores: 16
+ memory: 32768
+ hd: 102400
+csirtmu.jumbo16x64:
+ cores: 16
+ memory: 65536
+ hd: 102400
diff --git a/conf/router_attributes.yml b/conf/router_attributes.yml
new file mode 100644
index 0000000..18b02a8
--- /dev/null
+++ b/conf/router_attributes.yml
@@ -0,0 +1,3 @@
+base_box: generic/debian10
+memory: 256
+cpus: 1
diff --git a/conf/vagrant_mapping.yml b/conf/vagrant_mapping.yml
index cc903ad..3d7abf6 100644
--- a/conf/vagrant_mapping.yml
+++ b/conf/vagrant_mapping.yml
@@ -73,9 +73,3 @@ other:
vagrant_host: vagrant.host
vagrant_plugins: vagrant.plugins
vagrant_sensitive: vagrant.sensitive
-need_provider:
- flavor: flavor
- memory: memory
- cpus: cpus
-
-
diff --git a/conf/virtualbox_mapping.yml b/conf/virtualbox_mapping.yml
new file mode 100644
index 0000000..46e9575
--- /dev/null
+++ b/conf/virtualbox_mapping.yml
@@ -0,0 +1,3 @@
+integer:
+ memory: memory
+ cpus: cpus
diff --git a/create.py b/create.py
index 1b6c65f..845d52b 100644
--- a/create.py
+++ b/create.py
@@ -6,8 +6,9 @@ of virtual machines and network topology. See the documentation for details.
import sys
-from modules.file_generator import generate_vagrantfile, generate_ansible_files
-from modules.device_creator import open_file
+from modules.file_generator import generate_ansible_files # TODO change this to from modules.ansible_generator import generate_playbooks
+from modules.vagrant_generator import generate_vagrantfile
+from modules.file_manager import open_yaml
from modules.input_argument_parser import parse_input_args
from modules.input_file_validator import validate_device_definitions
from modules.preprocessing import preprocess
@@ -24,7 +25,7 @@ except Exception:
""" Parsing the definitions file. """
try:
- device_definitions = open_file(input_file_name)
+ device_definitions = open_yaml(input_file_name)
except Exception:
print("Definitions file could not be parsed.")
sys.exit(1)
diff --git a/modules/ansible_generator.py b/modules/ansible_generator.py
new file mode 100644
index 0000000..9d20533
--- /dev/null
+++ b/modules/ansible_generator.py
@@ -0,0 +1,6 @@
+def generate_playbooks(input_definitions, flags):
+ """ Generates ansible playbooks.
+
+ :param definitions: device definitions structure
+ :param flags: command line input flags
+ """
\ No newline at end of file
diff --git a/modules/file_generator.py b/modules/file_generator.py
index dc5c932..087a0cc 100644
--- a/modules/file_generator.py
+++ b/modules/file_generator.py
@@ -1,8 +1,6 @@
import jinja2
import os
-import conf.border_router
-from modules.device_creator import create_devices
from modules.ansible_data_generator import create_network_map, create_host_map, create_network_ips
def _load_template(template_name):
@@ -44,44 +42,12 @@ def _create_role_directory(role_name, provisioning_dir):
pass
-def _find_user_ansible_files(definitions):
- """ Finds the user ansible files and returns a list of host names. """
- host_names = []
-
- for host in definitions["hosts"]:
- if os.path.isfile("provisioning/" + host["name"] + ".yml" ):
- host_names.append(host["name"])
-
- return host_names
-
-def generate_vagrantfile(definitions, flags):
- """
- Creates Vagrantfile from definitions.
-
- :param definitions: device definitions structure
- :param flags: command line input flags
- """
-
- if "ansible_local" in flags and flags["ansible_local"]:
- ansible_local = True
- else:
- ansible_local = False
-
- device_definitions = create_devices(definitions, ansible_local)
- user_ansible_files = _find_user_ansible_files(definitions)
- template = _load_template("vagrantfile")
- output = template.render(devices=device_definitions, user_files=user_ansible_files, ansible_local=ansible_local)
- _generate_file("Vagrantfile", output)
-
- print("Info: Vagrantfile successfully created.")
-
-
def _generate_playbook(definitions):
""" Generates the main playbook. """
host_map = create_host_map(definitions["net_mappings"], definitions["router_mappings"], definitions["hosts"])
network = create_network_map(definitions)
-
+
template = _load_template("playbook")
output = template.render(hosts=host_map, routers=network)
@@ -97,7 +63,7 @@ def _generate_device_configuration(definitions):
""" Generates a playbook with basic device configutarion. """
host_map = create_host_map(definitions["net_mappings"], definitions["router_mappings"], definitions["hosts"])
- network = create_network_map(definitions)
+ network = create_network_map(definitions)
network_ips = create_network_ips(definitions["networks"])
template = _load_template("device_configuration")
@@ -123,7 +89,7 @@ def _generate_hosts_role(definitions):
_create_role_directory("hosts", "base_provisioning")
_generate_file("./base_provisioning/roles/hosts/tasks/main.yml", output)
-
+
user_template = _load_template("user_hosts")
user_output = template.render()
@@ -133,7 +99,7 @@ def _generate_hosts_role(definitions):
def _generate_separate_hosts_role(definitions):
""" Generate roles for separate host devices. """
-
+
host_map = create_host_map(definitions["net_mappings"], definitions["router_mappings"], definitions["hosts"])
for host in definitions["hosts"]:
@@ -166,7 +132,7 @@ def _generate_routers_role(definitions):
host_map = create_host_map(definitions["net_mappings"], definitions["router_mappings"], definitions["hosts"])
- network = create_network_map(definitions)
+ network = create_network_map(definitions)
template = _load_template("routers")
output = template.render(hosts=host_map, routers=network, border_router_ip=BORDER_ROUTER_IP)
@@ -201,8 +167,8 @@ def _generate_br_role(definitions):
print("Info: No router definition was found. Skipping border router creation.")
return
- network = create_network_map(definitions)
-
+ network = create_network_map(definitions)
+
host_map = create_host_map(definitions["net_mappings"], definitions["router_mappings"], definitions["hosts"])
routers_in_br_network = _get_br_routers(definitions)
@@ -220,8 +186,8 @@ def generate_ansible_files(definitions, flags):
:param definitions: device definitions structure
:param flags: command line input flags
- """
-
+ """
+
_generate_playbook(definitions)
_generate_device_configuration(definitions)
_generate_hosts_role(definitions)
diff --git a/modules/file_manager.py b/modules/file_manager.py
new file mode 100644
index 0000000..e732883
--- /dev/null
+++ b/modules/file_manager.py
@@ -0,0 +1,39 @@
+""" This module handles file imports and creations in general. """
+
+import jinja2
+import yaml
+
+def open_yaml(file_name):
+ """ Opens and returns a file from the argument. """
+ try:
+ input_file = open(str(file_name))
+ return yaml.safe_load(input_file)
+ except IOError:
+ print("Error: Cannot open the required file: " + str(file_name))
+ raise
+
+def generate_file(filename, output_string):
+ """
+ Generates a file from output string.
+
+ :param filename: name of the file to create
+ :param output_string: string to write to the file
+ """
+
+ try:
+ new_file = open(filename, "w")
+ new_file.write(output_string)
+ except IOError:
+ print("Error: cannot write to this location.")
+ raise
+
+def load_template(template_name):
+ """
+ Returns a loaded jinja2 template.
+
+ :param template_name: name of the template file
+ """
+
+ template_loader = jinja2.FileSystemLoader(searchpath="templates")
+ template_env = jinja2.Environment(loader=template_loader, trim_blocks=True, lstrip_blocks=True)
+ return template_env.get_template(template_name)
diff --git a/modules/preprocessing.py b/modules/preprocessing.py
index f21b6cc..cc70c65 100644
--- a/modules/preprocessing.py
+++ b/modules/preprocessing.py
@@ -3,6 +3,37 @@ called after validating the input but before device creation.
"""
from modules.border_router import create_border_router
+from modules.file_manager import open_yaml
+
+FLAVORS = open_yaml("conf/flavors.yml")
+ROUTER_ATTRIBUTES = open_yaml("conf/router_attributes.yml")
+
+
+def _configure_routers(definitions):
+ """ Adds predefined parameters to all routers if they are not defined in
+ the source yaml.
+ """
+
+ for router in definitions["routers"]:
+ for parameter, value in ROUTER_ATTRIBUTES.items():
+ if parameter not in router:
+ router[parameter] = value
+
+
+def _add_flavors(definitions):
+ """ Changes flavor attribute to cpus and memory. """
+
+ for host in definitions["hosts"]:
+ if "flavor" in host:
+ if host["flavor"] not in FLAVORS:
+ print("Error: Not supported flavor: " + host["flavor"])
+ raise AttributeError
+ if "memory" not in host:
+ host["memory"] = FLAVORS[host["flavor"]]["memory"]
+ if "cpus" not in host:
+ host["memory"] = FLAVORS[host["flavor"]]["cpus"]
+ host.pop("flavor")
+
def preprocess(definitions, flags):
"""
@@ -13,9 +44,20 @@ def preprocess(definitions, flags):
:param flags: a structure with command line flags
"""
- """ Creating Border router """
try:
create_border_router(definitions)
except Exception:
print("Could not create border router.")
raise
+
+ try:
+ _configure_routers(definitions)
+ except Exception:
+ print("Could not add router configurations to definitions.")
+ raise
+
+ try:
+ _add_flavors(definitions)
+ except Exception:
+ print("Could not add flavor.")
+ raise
diff --git a/modules/vagrant_generator.py b/modules/vagrant_generator.py
index f0c6ee8..1539b2e 100644
--- a/modules/vagrant_generator.py
+++ b/modules/vagrant_generator.py
@@ -1,10 +1,134 @@
""" This module generates a Vagrantfile from input device definitions. """
+import jinja2
-def _create_commands(device_name, device_type, input_definitions, flags):
+from modules.file_manager import load_template, generate_file, open_yaml
+
+VAGRANT_MAPPING = open_yaml("conf/vagrant_mapping.yml")
+VIRTUALBOX_MAPPING = open_yaml("conf/virtualbox_mapping.yml")
+
+
+def _create_simple_attribute(key, value, attribute_type):
+ """ Creates simple vagrant attributes like string, integer or boolean. """
+
+ attribute = dict()
+ attribute["type"] = attribute_type
+ attribute["command"] = key
+ attribute["value"] = value
+
+ return attribute
+
+
+def _create_commands(device_attributes, device_type, input_definitions, flags):
""" This function creates basic vagrant definition commands for a device. """
- # TODO create vagrant commands
+ commands = []
+ vb_commands = []
+
+ for attribute, value in device_attributes.items():
+ if attribute in VAGRANT_MAPPING["string"]:
+ commands.append(_create_simple_attribute(VAGRANT_MAPPING["string"][attribute], value, "string"))
+ elif attribute in VAGRANT_MAPPING["boolean"]:
+ commands.append(_create_simple_attribute(VAGRANT_MAPPING["boolean"][attribute], value, "boolean"))
+ elif attribute in VAGRANT_MAPPING["integer"]:
+ commands.append(_create_simple_attribute(VAGRANT_MAPPING["integer"][attribute], value, "integer"))
+ elif attribute in VIRTUALBOX_MAPPING["integer"]:
+ vb_commands.append(_create_simple_attribute(VIRTUALBOX_MAPPING["integer"][attribute], value, "integer"))
+
+ if vb_commands:
+ vb = dict()
+ vb["type"] = "provider"
+ vb["name"] = "virtualbox"
+ vb["commands"] = vb_commands
+ commands.append(vb)
+
+ return commands
+
+
+def _create_ansible_commands(playbook_location, flags):
+ """ Creates commands for running a playbook from the Vagrantfile. """
+
+ commands = []
+
+ playbook = dict()
+ playbook["type"] = "string"
+ playbook["command"] = "playbook"
+ playbook["value"] = playbook_location
+ commands.append(playbook)
+
+ if "verbose_ansible" in flags and flags["verbose_ansible"]:
+ verbosity = dict()
+ verbosity["type"] = "boolean"
+ verbosity["command"] = "verbose"
+ verbosity["value"] = True
+ commands.append(verbosity)
+
+ return commands
+
+
+def _find_netmask(network_name, networks):
+ """ Returns the netmask of a network address from network name. """
+
+ for network in networks:
+ if network['name'] == network_name:
+ address, netmask = network['cidr'].split('/')
+ return netmask
+
+
+def _add_networks_to_device(definition, mappings, input_definitions):
+ """ Adds networks to the vagrant definition of one device. """
+
+ for mapping in mappings:
+ if mapping[definition["type"]] == definition["name"]:
+ network = dict()
+ network["type"] = "network"
+ network["network_type"] = "private_network"
+ network["name"] = mapping["network"]
+ network["ip"] = mapping["ip"]
+ network["netmask"] = _find_netmask(mapping["network"], input_definitions["networks"])
+ definition["commands"].append(network)
+
+
+def _add_all_networks(vagrant_definitions, input_definitions, flags):
+ """ Adds all networks to vagrant definitions. """
+
+ for definition in vagrant_definitions:
+ if definition["type"] == "host":
+ _add_networks_to_device(definition, input_definitions["net_mappings"], input_definitions)
+ elif definition["type"] == "router":
+ _add_networks_to_device(definition, input_definitions["router_mappings"], input_definitions)
+
+
+def _call_provisioner(flags):
+ """ Creates entry to vagrant definitions for calling the provisioner. """
+
+ provisioner_calls = []
+
+ config_playbook = dict()
+ config_playbook["type"] = "provision"
+ if "ansible_local" in flags and flags["ansible_local"]:
+ config_playbook["provisioner"] = "ansible_local"
+ else:
+ config_playbook["provisioner"] = "ansible"
+ config_playbook["note"] = "basic configuration of devices and networks"
+ config_playbook["commands"] = _create_ansible_commands(
+ "base_provisioning/device_configuration.yml", flags)
+
+ provisioner_calls.append(config_playbook)
+
+ user_playbook = dict()
+ user_playbook["type"] = "provision"
+ if "ansible_local" in flags and flags["ansible_local"]:
+ user_playbook["provisioner"] = "ansible_local"
+ else:
+ user_playbook["provisioner"] = "ansible"
+ user_playbook["note"] = "user configuration of devices"
+ user_playbook["commands"] = _create_ansible_commands(
+ "base_provisioning/playbook.yml", flags)
+
+ provisioner_calls.append(user_playbook)
+
+ return provisioner_calls
def _build_vagrant_definitions(input_definitions, flags):
@@ -14,24 +138,39 @@ def _build_vagrant_definitions(input_definitions, flags):
"""
vagrant_definitions = []
-
for router in input_definitions["routers"]:
device = dict()
- device["device_name"] = router["name"]
- device["device_type"] = "router"
- device["commands"] = _create_commands(router["name"], "router", input_definitions, flags)
+ device["type"] = "router"
+ device["name"] = router["name"]
+ device["commands"] = _create_commands(router, "router", input_definitions, flags)
vagrant_definitions.append(device)
- for host in input_definitons["hosts"]:
+ for host in input_definitions["hosts"]:
device = dict()
- device["device_name"] = host["name"]
- device["device_type"] = "host"
- device["commands"] = _create_commands(host["name"], "host", input_definitions, flags)
+ device["type"] = "host"
+ device["name"] = host["name"]
+ device["commands"] = _create_commands(host, "host", input_definitions, flags)
vagrant_definitions.append(device)
-
+
+ _add_all_networks(vagrant_definitions, input_definitions, flags)
+
+ vagrant_definitions.extend(_call_provisioner(flags))
+
return vagrant_definitions
+def _build_vagrantfile(vagrant_definitions):
+ """
+ Generates the Vagrantfile using the vagrantfile template and vagrant
+ definitions.
+ """
+
+# TODO change newvagrantfile to vagrantfile
+ template = load_template("newvagrantfile")
+ output = template.render(defs=vagrant_definitions)
+ generate_file("Vagrantfile", output)
+
+
def generate_vagrantfile(input_definitions, flags):
"""
This method is responsible for Vagrantfile generation.
@@ -42,8 +181,12 @@ def generate_vagrantfile(input_definitions, flags):
try:
vagrant_definitions = _build_vagrant_definitions(input_definitions, flags)
- except Exception:
+ except Exception:
print("Could not create definitions for Vagrantfile.")
raise
- # TODO build Vagrantfile using a template
+ try:
+ _build_vagrantfile(vagrant_definitions)
+ except Exception:
+ print("Could not generate Vagrantfile.")
+ raise
diff --git a/templates/newvagrantfile b/templates/newvagrantfile
new file mode 100644
index 0000000..c87f2c6
--- /dev/null
+++ b/templates/newvagrantfile
@@ -0,0 +1,111 @@
+# Vagrantfile generated by Sandbox Creator.
+#
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+{# Macro for router items #}
+{% macro router(item, namespace) %}
+ # device (router): {{ item.name }}
+ {{ namespace }}.vm.define "{{ item.name }}" do |device|
+{{ layer2(item.commands, "device") }}
+{% endmacro -%}
+
+{# Macro for host items #}
+{% macro host(item, namespace) %}
+ # device (host): {{ item.name }}
+ {{ namespace }}.vm.define "{{ item.name }}" do |device|
+{{ layer2(item.commands, "device") }}
+{% endmacro -%}
+
+{# Macro for provider items #}
+{% macro provider(item, namespace) %}
+ {{ namespace }}.vm.provider "{{ item.name }}" do |provider|
+{{ layer3(item.commands, "provider") }}
+{% endmacro -%}
+
+{# Macro for provision items #}
+{% macro provision(item, namespace) %}
+ # {{ item.note }}
+ {{ namespace }}.vm.provision :{{ item.provisioner }} do |provisioner|
+{{ layer2(item.commands, "provisioner") }}
+{% endmacro -%}
+
+{# Macro for string items #}
+{% macro string(item, namespace) %}
+ {{ namespace }}.{{ item.command }} = "{{ item.value }}"
+{% endmacro -%}
+
+{# Macro for boolean items #}
+{% macro boolean(item, namespace) %}
+{% if item.value %}
+ {{ namespace }}.{{ item.command }} = true
+{% else %}
+ {{ namespace }}.{{ item.command }} = false
+{% endif %}
+{% endmacro -%}
+
+{# Macro for integer items #}
+{% macro integer(item, namespace) %}
+ {{ namespace }}.{{ item.command }} = {{ item.value }}
+{% endmacro -%}
+
+{# Macro for network items #}
+{% macro network(item, namespace) %}
+ {{ namespace }}.vm.network :{{ item.network_type }}, ip: "{{ item.ip }}"
+{%- if item.netmask %}
+, netmask: "{{ item.netmask }}"
+{%- endif %}
+{%- if item.network_type == "private_network" %}
+, virtualbox__intnet: "{{ item.name }}"
+{% endif %}
+{% endmacro -%}
+
+{# A macro that generates the first level of indentation. #}
+{% macro layer1(structure, namespace) %}
+{% for item in structure %}
+{% if item.type == "router" %}
+{{ router(item, namespace) }}
+{% elif item.type == "host" %}
+{{ host(item, namespace) }}
+{% elif item.type == "provision" %}
+{{ provision(item, namespace) }}
+{% endif %}
+{% endfor %}
+end
+{%- endmacro -%}
+
+{# A macro that generates the second level of indentation. #}
+{% macro layer2(structure, namespace) %}
+{% for item in structure %}
+{% if item.type == "string" %}
+ {{ string(item, namespace) -}}
+{% elif item.type == "boolean" %}
+ {{ boolean(item, namespace) -}}
+{% elif item.type == "integer" %}
+ {{ integer(item, namespace) -}}
+{% elif item.type == "provider" %}
+ {{ provider(item, namespace) -}}
+{% elif item.type == "network" %}
+ {{ network(item, namespace) -}}
+{% endif %}
+{% endfor %}
+ end
+{%- endmacro -%}
+
+{# A macro that generates the third level of indentation. #}
+{% macro layer3(structure, namespace) %}
+{% for item in structure %}
+{% if item.type == "string" %}
+ {{ string(item, namespace) -}}
+{% elif item.type == "boolean" %}
+ {{ boolean(item, namespace) -}}
+{% elif item.type == "integer" %}
+ {{ integer(item, namespace) -}}
+{% endif %}
+{% endfor %}
+ end
+{%- endmacro -%}
+
+Vagrant.configure("2") do |config|
+
+{{ layer1(defs, "config") -}}
--
GitLab