Skip to content
Snippets Groups Projects

Refactoring and extending flavor info generator

Merged Refactoring and extending flavor info generator
flavors_info_ext into master
Merged Jaromír Hradil requested to merge flavors_info_ext into master
Compare
and
2 files
+ 173
− 84
Compare changes
  • Side-by-side
  • Inline
Files
2
ci/flavor_info_miner.py
+ 172
− 83
import yaml
#!/usr/bin/env python3
import argparse
import csv
#Author: Jaromir Hradil 506487@mail.muni.cz
#This script gathers info from hiera about flavors
#Then it generates table in Markdown format.
#Setting arguments
parser = argparse.ArgumentParser(description='This script generates table with flavor info.')
parser.add_argument('--hiera-file', type=str, required=True, help='Path for hiera file flavors.yaml')
parser = parser.parse_args()
#Opening file
with open(parser.hiera_file,'r') as f:
data = yaml.safe_load(f)
data = data['cloud::profile::kolla::nova::controller::os_flavors']
#Names for columns
details = ['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average throughput (in MB per second)', 'GPU']
blacklist = ['cerit.hde-half', 'cerit.hde', 'meta.mema2', 'csirtmu.tiny1x2', 'csirtmu.tiny1x4', 'csirtmu.small2x4', 'csirtmu.small2x8', 'csirtmu.medium4x8', 'csirtmu.medium4x16',
'csirtmu.large8x16', 'csirtmu.large4x32', 'csirtmu.large8x32', 'csirtmu.jumbo16x32', 'csirtmu.jumbo8x64', 'csirtmu.jumbo16x64', 'du.perftest', 'admin.scaletest', 'hpc.18core-64ram-dukan',
'hpc.8core-32ram-dukan']
blacklist_dict = dict.fromkeys(blacklist)
 
"""
 
Script generates CSV output with available flavors
Actions performed:
final_out = []
generating of available flavors we offer
* if everything goes well
- image rotation news section is updated according to image rotation journal
#Printing specified data
for x in range(0,len(data)):
if(data[x]['name'] in blacklist_dict):
continue
if not ('aggregate_instance_extra_specs:hpc' in data[x]['extra_specs']):
hpc = 'No'
else:
hpc = 'No' if (data[x]['extra_specs']['aggregate_instance_extra_specs:hpc'] == 'false') else 'Yes'
if not ('aggregate_instance_extra_specs:ssd' in data[x]['extra_specs']):
ssd = 'No'
else:
ssd = 'No' if (data[x]['extra_specs']['aggregate_instance_extra_specs:ssd'] == 'false') else 'Yes'
gpu = 'No'
if ('pci_passthrough:alias' in data[x]['extra_specs']):
if(data[x]['extra_specs']['pci_passthrough:alias'].find('gpu') != -1):
gpu = 'Yes'
name = data[x]['name']
vcpus = data[x]['vcpus']
ram = data[x]['ram']/1024
#more readable format
disk = data[x]['extra_specs']['quota:disk_total_bytes_sec'] / 1000000 if 'quota:disk_total_bytes_sec' in data[x]['extra_specs'] else 'Unlimited'
net = data[x]['extra_specs']['quota:vif_outbound_average'] / 1000 if 'quota:vif_outbound_average' in data[x]['extra_specs'] else 'Unlimited'
iops = data[x]['extra_specs']['quota:disk_total_iops_sec'] if 'quota:disk_total_iops_sec' in data[x]['extra_specs'] else 'Unlimited'
#Printing final table sorted
final_out.append([f'{name}', f'{vcpus}', f'{ram:.0f}', f'{hpc}' , f'{ssd}',\
f'{disk}', f'{iops}', f'{net}', f'{gpu}'])
 
Usage:
 
python3 ./flavor_info_miner.py --hiera-file flavors.yaml
 
"""
 
import csv
 
import argparse
 
import yaml
with open('flavors.csv', 'w', newline='') as csvfile:
COLUMN_NAMES = [
"Flavor name",
writer = csv.DictWriter(csvfile, fieldnames=details)
"CPU",
writer.writeheader()
"RAM (in GB)",
"HPC",
for x in range(0,len(final_out)):
"SSD",
#['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average througput (in MB per second)', 'GPU']
"Disc throughput (in MB per second)",
writer.writerow({details[0] : final_out[x][0], details[1] : final_out[x][1], details[2] : final_out[x][2], details[3] : final_out[x][3],details[4] : final_out[x][4],\
"IOPS",
details[5] : final_out[x][5],details[6] : final_out[x][6],details[7] : final_out[x][7],details[8] : final_out[x][8]})
"Net average throughput (in MB per second)",
\ No newline at end of file
"GPU",
 
]
 
 
FLAVOR_BLACKLIST = [
 
"cerit.hde-half",
 
"cerit.hde",
 
"meta.mema2",
 
"csirtmu.tiny1x2",
 
"csirtmu.tiny1x4",
 
"csirtmu.small2x4",
 
"csirtmu.small2x8",
 
"csirtmu.medium4x8",
 
"csirtmu.medium4x16",
 
"csirtmu.large8x16",
 
"csirtmu.large4x32",
 
"csirtmu.large8x32",
 
"csirtmu.jumbo16x32",
 
"csirtmu.jumbo8x64",
 
"csirtmu.jumbo16x64",
 
"du.perftest",
 
"admin.scaletest",
 
"hpc.18core-64ram-dukan",
 
"hpc.8core-32ram-dukan",
 
]
 
 
 
def get_hiera_flavors(hiera_file):
 
"""Opens and extracts data from hiera file"""
 
with open(hiera_file, "r", encoding="utf-8") as hiera_file_h:
 
data = yaml.safe_load(hiera_file_h)
 
return data["cloud::profile::kolla::nova::controller::os_flavors"]
 
 
 
def filter_flavors(all_flavors):
 
"""Filters and excludes blacklisted and deprecated flavors"""
 
filtered_flavors = []
 
for flavor in all_flavors:
 
if (
 
flavor["name"] not in FLAVOR_BLACKLIST
 
and "deprecated" not in flavor["name"]
 
):
 
filtered_flavors.append(flavor)
 
return filtered_flavors
 
 
 
def generate_flavor_data(filtered_flavors):
 
"""Generates CSV output from extracted data"""
 
output = []
 
 
for flavor in filtered_flavors:
 
name = flavor["name"]
 
vcpus = flavor["vcpus"]
 
ram = flavor["ram"] / 1024
 
 
disk = (
 
flavor["extra_specs"]["quota:disk_total_bytes_sec"] / 1000000
 
if "quota:disk_total_bytes_sec" in flavor["extra_specs"]
 
else "Unlimited"
 
)
 
net = (
 
flavor["extra_specs"]["quota:vif_outbound_average"] / 1000
 
if "quota:vif_outbound_average" in flavor["extra_specs"]
 
else "Unlimited"
 
)
 
iops = (
 
flavor["extra_specs"]["quota:disk_total_iops_sec"]
 
if "quota:disk_total_iops_sec" in flavor["extra_specs"]
 
else "Unlimited"
 
)
 
 
if (
 
"aggregate_instance_extra_specs:hpc" not in flavor["extra_specs"]
 
or flavor["extra_specs"]["aggregate_instance_extra_specs:hpc"] == "false"
 
):
 
hpc = "No"
 
else:
 
hpc = "Yes"
 
 
if (
 
"aggregate_instance_extra_specs:ssd" not in flavor["extra_specs"]
 
or flavor["extra_specs"]["aggregate_instance_extra_specs:ssd"] == "false"
 
):
 
ssd = "No"
 
else:
 
ssd = "Yes"
 
 
if (
 
"pci_passthrough:alias" not in flavor["extra_specs"]
 
or "gpu" not in flavor["extra_specs"]["pci_passthrough:alias"]
 
):
 
gpu = "No"
 
else:
 
gpu = "Yes"
 
 
output.append(
 
[
 
f"{name}",
 
f"{vcpus}",
 
f"{ram:.0f}",
 
f"{hpc}",
 
f"{ssd}",
 
f"{disk}",
 
f"{iops}",
 
f"{net}",
 
f"{gpu}",
 
]
 
)
 
return output
 
 
 
def write_output_to_file(output, output_file):
 
"""Writed generated output into file"""
 
with open(output_file, "w", newline="") as csvfile:
 
 
writer = csv.DictWriter(csvfile, fieldnames=COLUMN_NAMES)
 
writer.writeheader()
 
 
for row in output:
 
# ['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average througput (in MB per second)', 'GPU']
 
writer.writerow(
 
{
 
COLUMN_NAMES[0]: row[0],
 
COLUMN_NAMES[1]: row[1],
 
COLUMN_NAMES[2]: row[2],
 
COLUMN_NAMES[3]: row[3],
 
COLUMN_NAMES[4]: row[4],
 
COLUMN_NAMES[5]: row[5],
 
COLUMN_NAMES[6]: row[6],
 
COLUMN_NAMES[7]: row[7],
 
COLUMN_NAMES[8]: row[8],
 
}
 
)
 
 
 
if __name__ == "__main__":
 
PARSER = argparse.ArgumentParser(
 
description="This script generates table with flavor info."
 
)
 
PARSER.add_argument(
 
"--hiera-file", type=str, required=True, help="Path for hiera file flavors.yaml"
 
)
 
PARSER.add_argument(
 
"--output-file", type=str, required=True, help="Path for output file"
 
)
 
ARGS = PARSER.parse_args()
 
 
ALL_FLAVORS = get_hiera_flavors(ARGS.hiera_file)
 
FILTERED_FLAVORS = filter_flavors(ALL_FLAVORS)
 
OUTPUT = generate_flavor_data(FILTERED_FLAVORS)
 
write_output_to_file(OUTPUT, ARGS.output_file)