Merge pull request #192 from stfc/weekly-reporting

ENH: Add bash script to get data and python script to format data

Weekly-Reporting/README.md

@@ -17,14 +17,15 @@ There is a template yaml file [here](data.yaml) which **requires all values to b
 
 ## Instructions:
 
-1. Fill in `data.yaml` with your reporting data.
+1. Source your openstack cli venv and OpenStack admin credentials.
+2. Run the `report.sh` script to generate the data file.
 2. Write your token in plain text in a file e.g. "token"
-2. Run `export.py` with the correct arguments, see below:
+3. Run `export.py` with the correct arguments, see below:
 
 ```shell
 python3 export.py --host="http://172.16.103.52:8086" \
 --org="cloud" \
 --bucket="weekly-reports-time"
 --token-file="token"
---report-file="data.yaml"
+--report-file="data-9.yaml"
 ```

Weekly-Reporting/export.py

@@ -1,4 +1,5 @@
 """Export weekly reports statistics data to an InfluxDB bucket."""
+
 from pprint import pprint
 from typing import Dict, List
 
@@ -30,11 +31,7 @@ def main(args: argparse.Namespace):
         points = _create_points_inventory(args.inventory_file)
     api_token = _get_token(args.token_file)
     _write_data(
-        points=points,
-        host=args.host,
-        org=args.org,
-        bucket=args.bucket,
-        token=api_token
+        points=points, host=args.host, org=args.org, bucket=args.bucket, token=api_token
     )
 
 
@@ -54,7 +51,9 @@ def _check_args(args: argparse.Namespace):
     if not args.report_file and not args.inventory_file:
         raise RuntimeError("Argument --report-file or --inventory-file not given.")
     if args.report_file and args.inventory_file:
-        raise RuntimeError("Argument --report-file and --inventory-file given. Only one data file can be provided.")
+        raise RuntimeError(
+            "Argument --report-file and --inventory-file given. Only one data file can be provided."
+        )
     if not Path(args.token_file).is_file():
         raise RuntimeError(f"Cannot find token file at path {args.token_file}.")
     if args.report_file and not Path(args.report_file).is_file():
@@ -126,18 +125,30 @@ def _from_key(key: str, data: Dict) -> List[Point]:
     if key == "virtual_worker_nodes":
         return _from_vwn(data)
     else:
-        raise RuntimeError(f"Key {key} not supported. Please contact service maintainer.")
+        raise RuntimeError(
+            f"Key {key} not supported. Please contact service maintainer."
+        )
 
 
 def _from_cpu(data: Dict) -> List[Point]:
     """Extract cpu data from yaml into a Point."""
-    return [Point("cpu").field("in_use", data["in_use"]).field("total", data["total"]).time(time)]
+    return [
+        Point("cpu")
+        .field("in_use", data["in_use"])
+        .field("total", data["total"])
+        .time(time)
+    ]
 
 
 def _from_memory(data: Dict) -> List[Point]:
     """Extract memory data from yaml into a Point."""
 
-    return [Point("memory").field("in_use", data["in_use"]).field("total", data["total"]).time(time)]
+    return [
+        Point("memory")
+        .field("in_use", data["in_use"])
+        .field("total", data["total"])
+        .time(time)
+    ]
 
 
 def _from_storage(data: Dict) -> List[Point]:
@@ -153,27 +164,41 @@ def _from_hv(data: Dict) -> List[Point]:
     """Extract hv data from yaml into Points."""
     points = []
     points.append(Point("hv").field("active", data["active"]["active"]).time(time))
-    points.append(Point("hv").field("active_and_cpu_full", data["active"]["cpu_full"]).time(time))
-    points.append(Point("hv").field("active_and_memory_full", data["active"]["memory_full"]).time(time))
+    points.append(
+        Point("hv").field("active_and_cpu_full", data["active"]["cpu_full"]).time(time)
+    )
+    points.append(
+        Point("hv")
+        .field("active_and_memory_full", data["active"]["memory_full"])
+        .time(time)
+    )
     points.append(Point("hv").field("down", data["down"]).time(time))
     points.append(Point("hv").field("disabled", data["disabled"]).time(time))
     return points
 
 
 def _from_vm(data: Dict) -> List[Point]:
     """Extract vm data from yaml into a Point."""
-    return [(
-        Point("vm")
-        .field("active", data["active"])
-        .field("shutoff", data["shutoff"])
-        .field("errored", data["errored"])
-        .field("building", data["building"])
-        .time(time))]
+    return [
+        (
+            Point("vm")
+            .field("active", data["active"])
+            .field("shutoff", data["shutoff"])
+            .field("errored", data["errored"])
+            .field("building", data["building"])
+            .time(time)
+        )
+    ]
 
 
 def _from_fip(data: Dict) -> List[Point]:
     """Extract floating ip data from yaml into a Point."""
-    return [Point("floating_ip").field("in_use", data["in_use"]).field("total", data["total"]).time(time)]
+    return [
+        Point("floating_ip")
+        .field("in_use", data["in_use"])
+        .field("total", data["total"])
+        .time(time)
+    ]
 
 
 def _from_vwn(data: Dict) -> List[Point]:
@@ -205,5 +230,3 @@ def _write_data(points: List[Point], host: str, org: str, bucket: str, token: st
     parser.add_argument("--inventory-file", help="Inventory ini file.")
     arguments = parser.parse_args()
     main(arguments)
-
-

Weekly-Reporting/format_raw_data.py

@@ -0,0 +1,79 @@
+import yaml
+from pprint import pprint
+
+import datetime
+
+with open("compute_service.yaml", "r", encoding="utf-8") as file:
+    compute_service = yaml.safe_load(file)
+
+with open("hypervisor.yaml", "r", encoding="utf-8") as file:
+    hypervisor = yaml.safe_load(file)
+
+with open("fip.yaml", "r", encoding="utf-8") as file:
+    fip = yaml.safe_load(file)
+
+with open("server.yaml", "r", encoding="utf-8") as file:
+    server = yaml.safe_load(file)
+
+
+hvs_enabled = [hv for hv in compute_service if hv["Status"] == "enabled"]
+hvs_up = len([hv for hv in hvs_enabled if hv["State"] == "up"])
+hvs_down = len([hv for hv in hvs_enabled if hv["State"] == "down"])
+hvs_disabled = len(
+    [hv for hv in hvs_enabled if hv["State"] == "up" and hv["Status"] == "disabled"]
+)
+
+memory_used = round(
+    sum([hv["Memory MB Used"] for hv in hypervisor if hv["State"] == "up"]) / 1000000, 2
+)
+memory_total = round(
+    sum([hv["Memory MB"] for hv in hypervisor if hv["State"] == "up"]) / 1000000, 2
+)
+
+cpu_used = sum([hv["vCPUs Used"] for hv in hypervisor if hv["State"] == "up"])
+cpu_total = sum([hv["vCPUs"] for hv in hypervisor if hv["State"] == "up"])
+
+hvs_up_and_cpu_full = len(
+    [hv for hv in hypervisor if hv["vCPUs Used"] == hv["vCPUs"] and hv["State"] == "up"]
+)
+hvs_up_and_memory_full = len(
+    [
+        hv
+        for hv in hypervisor
+        if (hv["Memory MB"] - hv["Memory MB Used"] <= 8192 and hv["State"] == "up")
+    ]
+)
+
+fip_total = fip["used_ips"]
+fip_used = fip["total_ips"]
+
+vm_active = len([vm for vm in server if vm["Status"] == "ACTIVE"])
+vm_error = len([vm for vm in server if vm["Status"] == "ERROR"])
+vm_build = len([vm for vm in server if vm["Status"] == "BUILD"])
+vm_shutoff = len([vm for vm in server if vm["Status"] == "SHUTOFF"])
+
+
+data = {}
+data["memory"] = {"in_use": memory_used, "total": memory_total}
+data["cpu"] = {"in_use": cpu_used, "total": cpu_total}
+data["hv"] = {
+    "active": {
+        "active": hvs_up,
+        "cpu_full": hvs_up_and_cpu_full,
+        "memory_ful": hvs_up_and_memory_full,
+    },
+    "down": hvs_down,
+    "disabled": hvs_disabled,
+}
+data["floating_ip"] = {"in_use": fip_used, "total": fip_total}
+data["vm"] = {
+    "active": vm_active,
+    "shutoff": vm_shutoff,
+    "errored": vm_error,
+    "building": vm_build,
+}
+
+with open(
+    f"data-{datetime.datetime.now().isocalendar().week}.yaml", "w", encoding="utf-8"
+) as file:
+    yaml.dump(data, file)

Weekly-Reporting/report.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+
+set -e
+
+openstack compute service list --long -f yaml > compute_service.yaml
+
+openstack hypervisor list --long -f yaml > hypervisor.yaml
+
+openstack ip availability show 0dc30001-edfb-4137-be76-8e51f38fd650 -f yaml > fip.yaml
+
+openstack server list --all-projects --limit -1 -f yaml > server.yaml
+
+
+python3 format_raw_data.py

Weekly-Reporting/requirements.txt

@@ -0,0 +1,2 @@
+python-openstackclient==5.8.1
+pyyaml