diff --git a/Weekly-Reporting/README.md b/Weekly-Reporting/README.md index 9b3441a2..dae4f9fc 100644 --- a/Weekly-Reporting/README.md +++ b/Weekly-Reporting/README.md @@ -17,14 +17,15 @@ There is a template yaml file [here](data.yaml) which **requires all values to b ## Instructions: -1. Fill in `data.yaml` with your reporting data. +1. Source your openstack cli venv and OpenStack admin credentials. +2. Run the `report.sh` script to generate the data file. 2. Write your token in plain text in a file e.g. "token" -2. Run `export.py` with the correct arguments, see below: +3. Run `export.py` with the correct arguments, see below: ```shell python3 export.py --host="http://172.16.103.52:8086" \ --org="cloud" \ --bucket="weekly-reports-time" --token-file="token" ---report-file="data.yaml" +--report-file="data-9.yaml" ``` diff --git a/Weekly-Reporting/data.yaml b/Weekly-Reporting/data.yaml.template similarity index 100% rename from Weekly-Reporting/data.yaml rename to Weekly-Reporting/data.yaml.template diff --git a/Weekly-Reporting/export.py b/Weekly-Reporting/export.py index 9ce369b7..3889fbc9 100644 --- a/Weekly-Reporting/export.py +++ b/Weekly-Reporting/export.py @@ -1,4 +1,5 @@ """Export weekly reports statistics data to an InfluxDB bucket.""" + from pprint import pprint from typing import Dict, List @@ -30,11 +31,7 @@ def main(args: argparse.Namespace): points = _create_points_inventory(args.inventory_file) api_token = _get_token(args.token_file) _write_data( - points=points, - host=args.host, - org=args.org, - bucket=args.bucket, - token=api_token + points=points, host=args.host, org=args.org, bucket=args.bucket, token=api_token ) @@ -54,7 +51,9 @@ def _check_args(args: argparse.Namespace): if not args.report_file and not args.inventory_file: raise RuntimeError("Argument --report-file or --inventory-file not given.") if args.report_file and args.inventory_file: - raise RuntimeError("Argument --report-file and --inventory-file given. Only one data file can be provided.") + raise RuntimeError( + "Argument --report-file and --inventory-file given. Only one data file can be provided." + ) if not Path(args.token_file).is_file(): raise RuntimeError(f"Cannot find token file at path {args.token_file}.") if args.report_file and not Path(args.report_file).is_file(): @@ -126,18 +125,30 @@ def _from_key(key: str, data: Dict) -> List[Point]: if key == "virtual_worker_nodes": return _from_vwn(data) else: - raise RuntimeError(f"Key {key} not supported. Please contact service maintainer.") + raise RuntimeError( + f"Key {key} not supported. Please contact service maintainer." + ) def _from_cpu(data: Dict) -> List[Point]: """Extract cpu data from yaml into a Point.""" - return [Point("cpu").field("in_use", data["in_use"]).field("total", data["total"]).time(time)] + return [ + Point("cpu") + .field("in_use", data["in_use"]) + .field("total", data["total"]) + .time(time) + ] def _from_memory(data: Dict) -> List[Point]: """Extract memory data from yaml into a Point.""" - return [Point("memory").field("in_use", data["in_use"]).field("total", data["total"]).time(time)] + return [ + Point("memory") + .field("in_use", data["in_use"]) + .field("total", data["total"]) + .time(time) + ] def _from_storage(data: Dict) -> List[Point]: @@ -153,8 +164,14 @@ def _from_hv(data: Dict) -> List[Point]: """Extract hv data from yaml into Points.""" points = [] points.append(Point("hv").field("active", data["active"]["active"]).time(time)) - points.append(Point("hv").field("active_and_cpu_full", data["active"]["cpu_full"]).time(time)) - points.append(Point("hv").field("active_and_memory_full", data["active"]["memory_full"]).time(time)) + points.append( + Point("hv").field("active_and_cpu_full", data["active"]["cpu_full"]).time(time) + ) + points.append( + Point("hv") + .field("active_and_memory_full", data["active"]["memory_full"]) + .time(time) + ) points.append(Point("hv").field("down", data["down"]).time(time)) points.append(Point("hv").field("disabled", data["disabled"]).time(time)) return points @@ -162,18 +179,26 @@ def _from_hv(data: Dict) -> List[Point]: def _from_vm(data: Dict) -> List[Point]: """Extract vm data from yaml into a Point.""" - return [( - Point("vm") - .field("active", data["active"]) - .field("shutoff", data["shutoff"]) - .field("errored", data["errored"]) - .field("building", data["building"]) - .time(time))] + return [ + ( + Point("vm") + .field("active", data["active"]) + .field("shutoff", data["shutoff"]) + .field("errored", data["errored"]) + .field("building", data["building"]) + .time(time) + ) + ] def _from_fip(data: Dict) -> List[Point]: """Extract floating ip data from yaml into a Point.""" - return [Point("floating_ip").field("in_use", data["in_use"]).field("total", data["total"]).time(time)] + return [ + Point("floating_ip") + .field("in_use", data["in_use"]) + .field("total", data["total"]) + .time(time) + ] def _from_vwn(data: Dict) -> List[Point]: @@ -205,5 +230,3 @@ def _write_data(points: List[Point], host: str, org: str, bucket: str, token: st parser.add_argument("--inventory-file", help="Inventory ini file.") arguments = parser.parse_args() main(arguments) - - diff --git a/Weekly-Reporting/format_raw_data.py b/Weekly-Reporting/format_raw_data.py new file mode 100644 index 00000000..f6729cd9 --- /dev/null +++ b/Weekly-Reporting/format_raw_data.py @@ -0,0 +1,79 @@ +import yaml +from pprint import pprint + +import datetime + +with open("compute_service.yaml", "r", encoding="utf-8") as file: + compute_service = yaml.safe_load(file) + +with open("hypervisor.yaml", "r", encoding="utf-8") as file: + hypervisor = yaml.safe_load(file) + +with open("fip.yaml", "r", encoding="utf-8") as file: + fip = yaml.safe_load(file) + +with open("server.yaml", "r", encoding="utf-8") as file: + server = yaml.safe_load(file) + + +hvs_enabled = [hv for hv in compute_service if hv["Status"] == "enabled"] +hvs_up = len([hv for hv in hvs_enabled if hv["State"] == "up"]) +hvs_down = len([hv for hv in hvs_enabled if hv["State"] == "down"]) +hvs_disabled = len( + [hv for hv in hvs_enabled if hv["State"] == "up" and hv["Status"] == "disabled"] +) + +memory_used = round( + sum([hv["Memory MB Used"] for hv in hypervisor if hv["State"] == "up"]) / 1000000, 2 +) +memory_total = round( + sum([hv["Memory MB"] for hv in hypervisor if hv["State"] == "up"]) / 1000000, 2 +) + +cpu_used = sum([hv["vCPUs Used"] for hv in hypervisor if hv["State"] == "up"]) +cpu_total = sum([hv["vCPUs"] for hv in hypervisor if hv["State"] == "up"]) + +hvs_up_and_cpu_full = len( + [hv for hv in hypervisor if hv["vCPUs Used"] == hv["vCPUs"] and hv["State"] == "up"] +) +hvs_up_and_memory_full = len( + [ + hv + for hv in hypervisor + if (hv["Memory MB"] - hv["Memory MB Used"] <= 8192 and hv["State"] == "up") + ] +) + +fip_total = fip["used_ips"] +fip_used = fip["total_ips"] + +vm_active = len([vm for vm in server if vm["Status"] == "ACTIVE"]) +vm_error = len([vm for vm in server if vm["Status"] == "ERROR"]) +vm_build = len([vm for vm in server if vm["Status"] == "BUILD"]) +vm_shutoff = len([vm for vm in server if vm["Status"] == "SHUTOFF"]) + + +data = {} +data["memory"] = {"in_use": memory_used, "total": memory_total} +data["cpu"] = {"in_use": cpu_used, "total": cpu_total} +data["hv"] = { + "active": { + "active": hvs_up, + "cpu_full": hvs_up_and_cpu_full, + "memory_ful": hvs_up_and_memory_full, + }, + "down": hvs_down, + "disabled": hvs_disabled, +} +data["floating_ip"] = {"in_use": fip_used, "total": fip_total} +data["vm"] = { + "active": vm_active, + "shutoff": vm_shutoff, + "errored": vm_error, + "building": vm_build, +} + +with open( + f"data-{datetime.datetime.now().isocalendar().week}.yaml", "w", encoding="utf-8" +) as file: + yaml.dump(data, file) diff --git a/Weekly-Reporting/report.sh b/Weekly-Reporting/report.sh new file mode 100644 index 00000000..7c2e8533 --- /dev/null +++ b/Weekly-Reporting/report.sh @@ -0,0 +1,15 @@ +#!/bin/bash + + +set -e + +openstack compute service list --long -f yaml > compute_service.yaml + +openstack hypervisor list --long -f yaml > hypervisor.yaml + +openstack ip availability show 0dc30001-edfb-4137-be76-8e51f38fd650 -f yaml > fip.yaml + +openstack server list --all-projects --limit -1 -f yaml > server.yaml + + +python3 format_raw_data.py \ No newline at end of file diff --git a/Weekly-Reporting/requirements.txt b/Weekly-Reporting/requirements.txt new file mode 100644 index 00000000..494990ae --- /dev/null +++ b/Weekly-Reporting/requirements.txt @@ -0,0 +1,2 @@ +python-openstackclient==5.8.1 +pyyaml \ No newline at end of file