Skip to content
Snippets Groups Projects
Unverified Commit 6b203128 authored by Jaromír Hradil's avatar Jaromír Hradil
Browse files

Refactoring and extending flavor info generator

parent ee6f7a92
No related branches found
No related tags found
1 merge request!97Refactoring and extending flavor info generator
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 1
− 1
View file @ 6b203128
......@@ -17,7 +17,7 @@ deploy_cpanel:
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ics.muni.cz/cloud/hiera.git
- sed -i '11,$d' content/cloud/flavors/index.md
- echo "{{< csv-table header=\"true\">}}" >> content/cloud/flavors/index.md
- python3 ci/flavor_info_miner.py --hiera-file ./hiera/group/ostack/flavors.yaml
- python3 ci/flavor_info_miner.py --hiera-file ./hiera/group/ostack/flavors.yaml --output-file flavors.csv
- cat flavors.csv >> content/cloud/flavors/index.md
- echo "{{</csv-table>}}" >> content/cloud/flavors/index.md
- cat content/cloud/flavors/index.md
......
ci/flavor_info_miner.py
+ 172
− 83
View file @ 6b203128
import yaml
import argparse
import csv
#Author: Jaromir Hradil 506487@mail.muni.cz
#This script gathers info from hiera about flavors
#Then it generates table in Markdown format.
#Setting arguments
parser = argparse.ArgumentParser(description='This script generates table with flavor info.')
parser.add_argument('--hiera-file', type=str, required=True, help='Path for hiera file flavors.yaml')
parser = parser.parse_args()
#Opening file
with open(parser.hiera_file,'r') as f:
data = yaml.safe_load(f)
data = data['cloud::profile::kolla::nova::controller::os_flavors']
#Names for columns
details = ['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average throughput (in MB per second)', 'GPU']
blacklist = ['cerit.hde-half', 'cerit.hde', 'meta.mema2', 'csirtmu.tiny1x2', 'csirtmu.tiny1x4', 'csirtmu.small2x4', 'csirtmu.small2x8', 'csirtmu.medium4x8', 'csirtmu.medium4x16',
'csirtmu.large8x16', 'csirtmu.large4x32', 'csirtmu.large8x32', 'csirtmu.jumbo16x32', 'csirtmu.jumbo8x64', 'csirtmu.jumbo16x64', 'du.perftest', 'admin.scaletest', 'hpc.18core-64ram-dukan',
'hpc.8core-32ram-dukan']
blacklist_dict = dict.fromkeys(blacklist)
#!/usr/bin/env python3
"""
Script generates CSV output with available flavors
final_out = []
#Printing specified data
for x in range(0,len(data)):
if(data[x]['name'] in blacklist_dict):
continue
if not ('aggregate_instance_extra_specs:hpc' in data[x]['extra_specs']):
hpc = 'No'
else:
hpc = 'No' if (data[x]['extra_specs']['aggregate_instance_extra_specs:hpc'] == 'false') else 'Yes'
if not ('aggregate_instance_extra_specs:ssd' in data[x]['extra_specs']):
ssd = 'No'
else:
ssd = 'No' if (data[x]['extra_specs']['aggregate_instance_extra_specs:ssd'] == 'false') else 'Yes'
gpu = 'No'
if ('pci_passthrough:alias' in data[x]['extra_specs']):
if(data[x]['extra_specs']['pci_passthrough:alias'].find('gpu') != -1):
gpu = 'Yes'
name = data[x]['name']
vcpus = data[x]['vcpus']
ram = data[x]['ram']/1024
#more readable format
disk = data[x]['extra_specs']['quota:disk_total_bytes_sec'] / 1000000 if 'quota:disk_total_bytes_sec' in data[x]['extra_specs'] else 'Unlimited'
net = data[x]['extra_specs']['quota:vif_outbound_average'] / 1000 if 'quota:vif_outbound_average' in data[x]['extra_specs'] else 'Unlimited'
iops = data[x]['extra_specs']['quota:disk_total_iops_sec'] if 'quota:disk_total_iops_sec' in data[x]['extra_specs'] else 'Unlimited'
#Printing final table sorted
final_out.append([f'{name}', f'{vcpus}', f'{ram:.0f}', f'{hpc}' , f'{ssd}',\
f'{disk}', f'{iops}', f'{net}', f'{gpu}'])
Actions performed:
generating of available flavors we offer
* if everything goes well
- image rotation news section is updated according to image rotation journal
Usage:
python3 ./flavor_info_miner.py --hiera-file flavors.yaml
"""
import csv
import argparse
import yaml
with open('flavors.csv', 'w', newline='') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=details)
writer.writeheader()
for x in range(0,len(final_out)):
#['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average througput (in MB per second)', 'GPU']
writer.writerow({details[0] : final_out[x][0], details[1] : final_out[x][1], details[2] : final_out[x][2], details[3] : final_out[x][3],details[4] : final_out[x][4],\
details[5] : final_out[x][5],details[6] : final_out[x][6],details[7] : final_out[x][7],details[8] : final_out[x][8]})
\ No newline at end of file
COLUMN_NAMES = [
"Flavor name",
"CPU",
"RAM (in GB)",
"HPC",
"SSD",
"Disc throughput (in MB per second)",
"IOPS",
"Net average throughput (in MB per second)",
"GPU",
]
FLAVOR_BLACKLIST = [
"cerit.hde-half",
"cerit.hde",
"meta.mema2",
"csirtmu.tiny1x2",
"csirtmu.tiny1x4",
"csirtmu.small2x4",
"csirtmu.small2x8",
"csirtmu.medium4x8",
"csirtmu.medium4x16",
"csirtmu.large8x16",
"csirtmu.large4x32",
"csirtmu.large8x32",
"csirtmu.jumbo16x32",
"csirtmu.jumbo8x64",
"csirtmu.jumbo16x64",
"du.perftest",
"admin.scaletest",
"hpc.18core-64ram-dukan",
"hpc.8core-32ram-dukan",
]
def get_hiera_flavors(hiera_file):
"""Opens and extracts data from hiera file"""
with open(hiera_file, "r", encoding="utf-8") as hiera_file_h:
data = yaml.safe_load(hiera_file_h)
return data["cloud::profile::kolla::nova::controller::os_flavors"]
def filter_flavors(all_flavors):
"""Filters and excludes blacklisted and deprecated flavors"""
filtered_flavors = []
for flavor in all_flavors:
if (
flavor["name"] not in FLAVOR_BLACKLIST
and "deprecated" not in flavor["name"]
):
filtered_flavors.append(flavor)
return filtered_flavors
def generate_flavor_data(filtered_flavors):
"""Generates CSV output from extracted data"""
output = []
for flavor in filtered_flavors:
name = flavor["name"]
vcpus = flavor["vcpus"]
ram = flavor["ram"] / 1024
disk = (
flavor["extra_specs"]["quota:disk_total_bytes_sec"] / 1000000
if "quota:disk_total_bytes_sec" in flavor["extra_specs"]
else "Unlimited"
)
net = (
flavor["extra_specs"]["quota:vif_outbound_average"] / 1000
if "quota:vif_outbound_average" in flavor["extra_specs"]
else "Unlimited"
)
iops = (
flavor["extra_specs"]["quota:disk_total_iops_sec"]
if "quota:disk_total_iops_sec" in flavor["extra_specs"]
else "Unlimited"
)
if (
"aggregate_instance_extra_specs:hpc" not in flavor["extra_specs"]
or flavor["extra_specs"]["aggregate_instance_extra_specs:hpc"] == "false"
):
hpc = "No"
else:
hpc = "Yes"
if (
"aggregate_instance_extra_specs:ssd" not in flavor["extra_specs"]
or flavor["extra_specs"]["aggregate_instance_extra_specs:ssd"] == "false"
):
ssd = "No"
else:
ssd = "Yes"
if (
"pci_passthrough:alias" not in flavor["extra_specs"]
or "gpu" not in flavor["extra_specs"]["pci_passthrough:alias"]
):
gpu = "No"
else:
gpu = "Yes"
output.append(
[
f"{name}",
f"{vcpus}",
f"{ram:.0f}",
f"{hpc}",
f"{ssd}",
f"{disk}",
f"{iops}",
f"{net}",
f"{gpu}",
]
)
return output
def write_output_to_file(output, output_file):
"""Writed generated output into file"""
with open(output_file, "w", newline="") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=COLUMN_NAMES)
writer.writeheader()
for row in output:
# ['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average througput (in MB per second)', 'GPU']
writer.writerow(
{
COLUMN_NAMES[0]: row[0],
COLUMN_NAMES[1]: row[1],
COLUMN_NAMES[2]: row[2],
COLUMN_NAMES[3]: row[3],
COLUMN_NAMES[4]: row[4],
COLUMN_NAMES[5]: row[5],
COLUMN_NAMES[6]: row[6],
COLUMN_NAMES[7]: row[7],
COLUMN_NAMES[8]: row[8],
}
)
if __name__ == "__main__":
PARSER = argparse.ArgumentParser(
description="This script generates table with flavor info."
)
PARSER.add_argument(
"--hiera-file", type=str, required=True, help="Path for hiera file flavors.yaml"
)
PARSER.add_argument(
"--output-file", type=str, required=True, help="Path for output file"
)
ARGS = PARSER.parse_args()
ALL_FLAVORS = get_hiera_flavors(ARGS.hiera_file)
FILTERED_FLAVORS = filter_flavors(ALL_FLAVORS)
OUTPUT = generate_flavor_data(FILTERED_FLAVORS)
write_output_to_file(OUTPUT, ARGS.output_file)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment