Skip to content
Snippets Groups Projects

Refactoring and extending flavor info generator

Merged Refactoring and extending flavor info generator
flavors_info_ext into master
Merged Jaromír Hradil requested to merge flavors_info_ext into master
Compare
and
2 files
+ 173
− 84
Compare changes
  • Side-by-side
  • Inline
Files
2
ci/flavor_info_miner.py
+ 172
− 83
import yaml
import argparse
import csv
#Author: Jaromir Hradil 506487@mail.muni.cz
#This script gathers info from hiera about flavors
#Then it generates table in Markdown format.
#Setting arguments
parser = argparse.ArgumentParser(description='This script generates table with flavor info.')
parser.add_argument('--hiera-file', type=str, required=True, help='Path for hiera file flavors.yaml')
parser = parser.parse_args()
#Opening file
with open(parser.hiera_file,'r') as f:
data = yaml.safe_load(f)
data = data['cloud::profile::kolla::nova::controller::os_flavors']
#Names for columns
details = ['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average throughput (in MB per second)', 'GPU']
blacklist = ['cerit.hde-half', 'cerit.hde', 'meta.mema2', 'csirtmu.tiny1x2', 'csirtmu.tiny1x4', 'csirtmu.small2x4', 'csirtmu.small2x8', 'csirtmu.medium4x8', 'csirtmu.medium4x16',
'csirtmu.large8x16', 'csirtmu.large4x32', 'csirtmu.large8x32', 'csirtmu.jumbo16x32', 'csirtmu.jumbo8x64', 'csirtmu.jumbo16x64', 'du.perftest', 'admin.scaletest', 'hpc.18core-64ram-dukan',
'hpc.8core-32ram-dukan']
blacklist_dict = dict.fromkeys(blacklist)
#!/usr/bin/env python3
"""
Script generates CSV output with available flavors
final_out = []
#Printing specified data
for x in range(0,len(data)):
if(data[x]['name'] in blacklist_dict):
continue
if not ('aggregate_instance_extra_specs:hpc' in data[x]['extra_specs']):
hpc = 'No'
else:
hpc = 'No' if (data[x]['extra_specs']['aggregate_instance_extra_specs:hpc'] == 'false') else 'Yes'
if not ('aggregate_instance_extra_specs:ssd' in data[x]['extra_specs']):
ssd = 'No'
else:
ssd = 'No' if (data[x]['extra_specs']['aggregate_instance_extra_specs:ssd'] == 'false') else 'Yes'
gpu = 'No'
if ('pci_passthrough:alias' in data[x]['extra_specs']):
if(data[x]['extra_specs']['pci_passthrough:alias'].find('gpu') != -1):
gpu = 'Yes'
name = data[x]['name']
vcpus = data[x]['vcpus']
ram = data[x]['ram']/1024
#more readable format
disk = data[x]['extra_specs']['quota:disk_total_bytes_sec'] / 1000000 if 'quota:disk_total_bytes_sec' in data[x]['extra_specs'] else 'Unlimited'
net = data[x]['extra_specs']['quota:vif_outbound_average'] / 1000 if 'quota:vif_outbound_average' in data[x]['extra_specs'] else 'Unlimited'
iops = data[x]['extra_specs']['quota:disk_total_iops_sec'] if 'quota:disk_total_iops_sec' in data[x]['extra_specs'] else 'Unlimited'
#Printing final table sorted
final_out.append([f'{name}', f'{vcpus}', f'{ram:.0f}', f'{hpc}' , f'{ssd}',\
f'{disk}', f'{iops}', f'{net}', f'{gpu}'])
Actions performed:
generating of available flavors we offer
* if everything goes well
- image rotation news section is updated according to image rotation journal
Usage:
python3 ./flavor_info_miner.py --hiera-file flavors.yaml
"""
import csv
import argparse
import yaml
with open('flavors.csv', 'w', newline='') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=details)
writer.writeheader()
for x in range(0,len(final_out)):
#['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average througput (in MB per second)', 'GPU']
writer.writerow({details[0] : final_out[x][0], details[1] : final_out[x][1], details[2] : final_out[x][2], details[3] : final_out[x][3],details[4] : final_out[x][4],\
details[5] : final_out[x][5],details[6] : final_out[x][6],details[7] : final_out[x][7],details[8] : final_out[x][8]})
\ No newline at end of file
COLUMN_NAMES = [
"Flavor name",
"CPU",
"RAM (in GB)",
"HPC",
"SSD",
"Disc throughput (in MB per second)",
"IOPS",
"Net average throughput (in MB per second)",
"GPU",
]
FLAVOR_BLACKLIST = [
"cerit.hde-half",
"cerit.hde",
"meta.mema2",
"csirtmu.tiny1x2",
"csirtmu.tiny1x4",
"csirtmu.small2x4",
"csirtmu.small2x8",
"csirtmu.medium4x8",
"csirtmu.medium4x16",
"csirtmu.large8x16",
"csirtmu.large4x32",
"csirtmu.large8x32",
"csirtmu.jumbo16x32",
"csirtmu.jumbo8x64",
"csirtmu.jumbo16x64",
"du.perftest",
"admin.scaletest",
"hpc.18core-64ram-dukan",
"hpc.8core-32ram-dukan",
]
def get_hiera_flavors(hiera_file):
"""Opens and extracts data from hiera file"""
with open(hiera_file, "r", encoding="utf-8") as hiera_file_h:
data = yaml.safe_load(hiera_file_h)
return data["cloud::profile::kolla::nova::controller::os_flavors"]
def filter_flavors(all_flavors):
"""Filters and excludes blacklisted and deprecated flavors"""
filtered_flavors = []
for flavor in all_flavors:
if (
flavor["name"] not in FLAVOR_BLACKLIST
and "deprecated" not in flavor["name"]
):
filtered_flavors.append(flavor)
return filtered_flavors
def generate_flavor_data(filtered_flavors):
"""Generates CSV output from extracted data"""
output = []
for flavor in filtered_flavors:
name = flavor["name"]
vcpus = flavor["vcpus"]
ram = flavor["ram"] / 1024
disk = (
flavor["extra_specs"]["quota:disk_total_bytes_sec"] / 1000000
if "quota:disk_total_bytes_sec" in flavor["extra_specs"]
else "Unlimited"
)
net = (
flavor["extra_specs"]["quota:vif_outbound_average"] / 1000
if "quota:vif_outbound_average" in flavor["extra_specs"]
else "Unlimited"
)
iops = (
flavor["extra_specs"]["quota:disk_total_iops_sec"]
if "quota:disk_total_iops_sec" in flavor["extra_specs"]
else "Unlimited"
)
if (
"aggregate_instance_extra_specs:hpc" not in flavor["extra_specs"]
or flavor["extra_specs"]["aggregate_instance_extra_specs:hpc"] == "false"
):
hpc = "No"
else:
hpc = "Yes"
if (
"aggregate_instance_extra_specs:ssd" not in flavor["extra_specs"]
or flavor["extra_specs"]["aggregate_instance_extra_specs:ssd"] == "false"
):
ssd = "No"
else:
ssd = "Yes"
if (
"pci_passthrough:alias" not in flavor["extra_specs"]
or "gpu" not in flavor["extra_specs"]["pci_passthrough:alias"]
):
gpu = "No"
else:
gpu = "Yes"
output.append(
[
f"{name}",
f"{vcpus}",
f"{ram:.0f}",
f"{hpc}",
f"{ssd}",
f"{disk}",
f"{iops}",
f"{net}",
f"{gpu}",
]
)
return output
def write_output_to_file(output, output_file):
"""Writed generated output into file"""
with open(output_file, "w", newline="") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=COLUMN_NAMES)
writer.writeheader()
for row in output:
# ['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average througput (in MB per second)', 'GPU']
writer.writerow(
{
COLUMN_NAMES[0]: row[0],
COLUMN_NAMES[1]: row[1],
COLUMN_NAMES[2]: row[2],
COLUMN_NAMES[3]: row[3],
COLUMN_NAMES[4]: row[4],
COLUMN_NAMES[5]: row[5],
COLUMN_NAMES[6]: row[6],
COLUMN_NAMES[7]: row[7],
COLUMN_NAMES[8]: row[8],
}
)
if __name__ == "__main__":
PARSER = argparse.ArgumentParser(
description="This script generates table with flavor info."
)
PARSER.add_argument(
"--hiera-file", type=str, required=True, help="Path for hiera file flavors.yaml"
)
PARSER.add_argument(
"--output-file", type=str, required=True, help="Path for output file"
)
ARGS = PARSER.parse_args()
ALL_FLAVORS = get_hiera_flavors(ARGS.hiera_file)
FILTERED_FLAVORS = filter_flavors(ALL_FLAVORS)
OUTPUT = generate_flavor_data(FILTERED_FLAVORS)
write_output_to_file(OUTPUT, ARGS.output_file)