Merge branch 'flavors_info_ext' into 'master'

Refactoring and extending flavor info generator

See merge request !97

.gitlab-ci.yml

@@ -17,7 +17,7 @@ deploy_cpanel:
   - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ics.muni.cz/cloud/hiera.git  
   - sed -i '11,$d' content/cloud/flavors/index.md 
   - echo "{{< csv-table header=\"true\">}}" >> content/cloud/flavors/index.md
-  - python3 ci/flavor_info_miner.py --hiera-file ./hiera/group/ostack/flavors.yaml
+  - python3 ci/flavor_info_miner.py --hiera-file ./hiera/group/ostack/flavors.yaml --output-file flavors.csv
   - cat flavors.csv >> content/cloud/flavors/index.md
   - echo "{{</csv-table>}}" >> content/cloud/flavors/index.md
   - cat content/cloud/flavors/index.md

ci/flavor_info_miner.py

-import yaml
-import argparse
-import csv
-
-
-#Author: Jaromir Hradil 506487@mail.muni.cz
-
-#This script gathers info from hiera about flavors
-#Then it generates table in Markdown format.
-
-#Setting arguments
-parser = argparse.ArgumentParser(description='This script generates table with flavor info.')
-
-parser.add_argument('--hiera-file', type=str, required=True, help='Path for hiera file flavors.yaml')
-
-parser = parser.parse_args()
-
-#Opening file
-with open(parser.hiera_file,'r') as f:
-    data = yaml.safe_load(f)
-
-
-data = data['cloud::profile::kolla::nova::controller::os_flavors']
-
-
-
-#Names for columns
-details = ['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average throughput (in MB per second)', 'GPU']
-
-blacklist = ['cerit.hde-half', 'cerit.hde', 'meta.mema2', 'csirtmu.tiny1x2', 'csirtmu.tiny1x4', 'csirtmu.small2x4', 'csirtmu.small2x8', 'csirtmu.medium4x8', 'csirtmu.medium4x16',
-'csirtmu.large8x16', 'csirtmu.large4x32', 'csirtmu.large8x32', 'csirtmu.jumbo16x32', 'csirtmu.jumbo8x64', 'csirtmu.jumbo16x64', 'du.perftest', 'admin.scaletest', 'hpc.18core-64ram-dukan',
-'hpc.8core-32ram-dukan']
-
-blacklist_dict = dict.fromkeys(blacklist)
+#!/usr/bin/env python3
 
+"""
+Script generates CSV output with available flavors
 
-
-final_out = []
-
-
-#Printing specified data
-for x in range(0,len(data)):
-
-
-    if(data[x]['name'] in blacklist_dict):
-        continue
-
-    if not ('aggregate_instance_extra_specs:hpc' in data[x]['extra_specs']):
-        hpc = 'No'
-    else:
-        hpc = 'No' if (data[x]['extra_specs']['aggregate_instance_extra_specs:hpc'] == 'false') else 'Yes'
-
-    if not ('aggregate_instance_extra_specs:ssd' in data[x]['extra_specs']):
-        ssd = 'No'
-    else:
-        ssd = 'No' if (data[x]['extra_specs']['aggregate_instance_extra_specs:ssd'] == 'false') else 'Yes'
-
-    gpu = 'No'
-
-    if ('pci_passthrough:alias' in data[x]['extra_specs']):
-
-        if(data[x]['extra_specs']['pci_passthrough:alias'].find('gpu') != -1):
-            gpu = 'Yes'
-
-    name = data[x]['name']
-    vcpus = data[x]['vcpus']
-    ram = data[x]['ram']/1024
-
-    #more readable format
-    disk = data[x]['extra_specs']['quota:disk_total_bytes_sec'] / 1000000 if 'quota:disk_total_bytes_sec' in data[x]['extra_specs'] else 'Unlimited'
-    net =  data[x]['extra_specs']['quota:vif_outbound_average'] / 1000  if 'quota:vif_outbound_average' in data[x]['extra_specs'] else 'Unlimited'
-    iops =  data[x]['extra_specs']['quota:disk_total_iops_sec'] if 'quota:disk_total_iops_sec' in data[x]['extra_specs'] else 'Unlimited'
-
-    #Printing final table sorted
-    final_out.append([f'{name}', f'{vcpus}',  f'{ram:.0f}', f'{hpc}' , f'{ssd}',\
-          f'{disk}', f'{iops}', f'{net}',  f'{gpu}'])
+Actions performed:
+ generating of available flavors we offer 
+ * if everything goes well
+   - image rotation news section is updated according to image rotation journal
 
 
+Usage:
+  python3 ./flavor_info_miner.py --hiera-file flavors.yaml
+"""
+import csv
+import argparse
 
+import yaml
 
-with open('flavors.csv', 'w', newline='') as csvfile:
-    
-    writer = csv.DictWriter(csvfile, fieldnames=details)
-    writer.writeheader()
-    
-    for x in range(0,len(final_out)):
-#['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average througput (in MB per second)', 'GPU']
-        writer.writerow({details[0] : final_out[x][0], details[1] : final_out[x][1], details[2] : final_out[x][2], details[3] : final_out[x][3],details[4] : final_out[x][4],\
-            details[5] : final_out[x][5],details[6] : final_out[x][6],details[7] : final_out[x][7],details[8] : final_out[x][8]})
\ No newline at end of file
+COLUMN_NAMES = [
+    "Flavor name",
+    "CPU",
+    "RAM (in GB)",
+    "HPC",
+    "SSD",
+    "Disc throughput (in MB per second)",
+    "IOPS",
+    "Net average throughput (in MB per second)",
+    "GPU",
+]
+
+FLAVOR_BLACKLIST = [
+    "cerit.hde-half",
+    "cerit.hde",
+    "meta.mema2",
+    "csirtmu.tiny1x2",
+    "csirtmu.tiny1x4",
+    "csirtmu.small2x4",
+    "csirtmu.small2x8",
+    "csirtmu.medium4x8",
+    "csirtmu.medium4x16",
+    "csirtmu.large8x16",
+    "csirtmu.large4x32",
+    "csirtmu.large8x32",
+    "csirtmu.jumbo16x32",
+    "csirtmu.jumbo8x64",
+    "csirtmu.jumbo16x64",
+    "du.perftest",
+    "admin.scaletest",
+    "hpc.18core-64ram-dukan",
+    "hpc.8core-32ram-dukan",
+]
+
+
+def get_hiera_flavors(hiera_file):
+    """Opens and extracts data from hiera file"""
+    with open(hiera_file, "r", encoding="utf-8") as hiera_file_h:
+        data = yaml.safe_load(hiera_file_h)
+        return data["cloud::profile::kolla::nova::controller::os_flavors"]
+
+
+def filter_flavors(all_flavors):
+    """Filters and excludes blacklisted and deprecated flavors"""
+    filtered_flavors = []
+    for flavor in all_flavors:
+        if (
+            flavor["name"] not in FLAVOR_BLACKLIST
+            and "deprecated" not in flavor["name"]
+        ):
+            filtered_flavors.append(flavor)
+    return filtered_flavors
+
+
+def generate_flavor_data(filtered_flavors):
+    """Generates CSV output from extracted data"""
+    output = []
+
+    for flavor in filtered_flavors:
+        name = flavor["name"]
+        vcpus = flavor["vcpus"]
+        ram = flavor["ram"] / 1024
+
+        disk = (
+            flavor["extra_specs"]["quota:disk_total_bytes_sec"] / 1000000
+            if "quota:disk_total_bytes_sec" in flavor["extra_specs"]
+            else "Unlimited"
+        )
+        net = (
+            flavor["extra_specs"]["quota:vif_outbound_average"] / 1000
+            if "quota:vif_outbound_average" in flavor["extra_specs"]
+            else "Unlimited"
+        )
+        iops = (
+            flavor["extra_specs"]["quota:disk_total_iops_sec"]
+            if "quota:disk_total_iops_sec" in flavor["extra_specs"]
+            else "Unlimited"
+        )
+
+        if (
+            "aggregate_instance_extra_specs:hpc" not in flavor["extra_specs"]
+            or flavor["extra_specs"]["aggregate_instance_extra_specs:hpc"] == "false"
+        ):
+            hpc = "No"
+        else:
+            hpc = "Yes"
+
+        if (
+            "aggregate_instance_extra_specs:ssd" not in flavor["extra_specs"]
+            or flavor["extra_specs"]["aggregate_instance_extra_specs:ssd"] == "false"
+        ):
+            ssd = "No"
+        else:
+            ssd = "Yes"
+
+        if (
+            "pci_passthrough:alias" not in flavor["extra_specs"]
+            or "gpu" not in flavor["extra_specs"]["pci_passthrough:alias"]
+        ):
+            gpu = "No"
+        else:
+            gpu = "Yes"
+
+        output.append(
+            [
+                f"{name}",
+                f"{vcpus}",
+                f"{ram:.0f}",
+                f"{hpc}",
+                f"{ssd}",
+                f"{disk}",
+                f"{iops}",
+                f"{net}",
+                f"{gpu}",
+            ]
+        )
+    return output
+
+
+def write_output_to_file(output, output_file):
+    """Writed generated output into file"""
+    with open(output_file, "w", newline="") as csvfile:
+
+        writer = csv.DictWriter(csvfile, fieldnames=COLUMN_NAMES)
+        writer.writeheader()
+
+        for row in output:
+            # ['Flavor name' , 'CPU' , 'RAM (in GB)' , 'HPC' , 'SSD' , 'Disc throughput (in MB per second)' , 'IOPS', 'Net average througput (in MB per second)', 'GPU']
+            writer.writerow(
+                {
+                    COLUMN_NAMES[0]: row[0],
+                    COLUMN_NAMES[1]: row[1],
+                    COLUMN_NAMES[2]: row[2],
+                    COLUMN_NAMES[3]: row[3],
+                    COLUMN_NAMES[4]: row[4],
+                    COLUMN_NAMES[5]: row[5],
+                    COLUMN_NAMES[6]: row[6],
+                    COLUMN_NAMES[7]: row[7],
+                    COLUMN_NAMES[8]: row[8],
+                }
+            )
+
+
+if __name__ == "__main__":
+    PARSER = argparse.ArgumentParser(
+        description="This script generates table with flavor info."
+    )
+    PARSER.add_argument(
+        "--hiera-file", type=str, required=True, help="Path for hiera file flavors.yaml"
+    )
+    PARSER.add_argument(
+        "--output-file", type=str, required=True, help="Path for output file"
+    )
+    ARGS = PARSER.parse_args()
+
+    ALL_FLAVORS = get_hiera_flavors(ARGS.hiera_file)
+    FILTERED_FLAVORS = filter_flavors(ALL_FLAVORS)
+    OUTPUT = generate_flavor_data(FILTERED_FLAVORS)
+    write_output_to_file(OUTPUT, ARGS.output_file)