diff --git a/AUTHORS b/AUTHORS index 0461765..90142e0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,7 +1,6 @@ -All the team members and collaborators -that touched this project somehow will be added to the list -below: +All collaborators with this project: -Team Members: +Collaborators: ------------- -- Eduardo Cerqueira +- Eduardo Cerqueira +- Pavol Kotvan diff --git a/Makefile b/Makefile index a080595..09ba3c1 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,10 @@ default: help NAME=janitor MAN=janitor.1 -VERSION=0.1 +VERSION=0.2 RPMDIST=$(shell rpm --eval '%dist') #RELEASE=1$(rpmsuffix)$(RPMDIST) -RELEASE=1 +RELEASE=0 PWD=$(shell bash -c "pwd -P") RPMTOP=$(PWD)/rpmbuild SPEC=$(NAME).spec @@ -18,7 +18,7 @@ help: @echo @echo "clean clean temp files from local workspace" @echo "doc generate sphinx documentation and man pages" - @echo "test run unit tests locally" + @echo "test run functional/unit tests locally" @echo "tarball generate tarball of p roject" @echo "rpm build source codes and generate rpm file" @echo "srpm generate SRPM file" diff --git a/README.md b/README.md index 32f15f6..0a4fce0 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,193 @@ ![copr build](https://copr.fedorainfracloud.org/coprs/eduardocerqueira/janitor/package/janitor/status_image/last_build.png) -# janitor +# Janitor -MOTIVATION ----------- + -One of my Openstack tenants that I use to run tests for an internal tool has a very limited quota so all the time I need to stop my tests and automations and spend few minutes -cleaning up public/floating ips, destroying virtual machines and others. The idea to have janitor or a butler is actually an old idea and a coworker had implemented -it but in a more sophisticated and fashion way this is a simply version and currently only focusing in Openstack maybe further I'll extend it to AWS, Openshift and others. +Janitor is a Linux component to perform quota management tasks to an [Openstack](https://www.openstack.org/) tenant. -WHAT IS THIS -------------- +It `is not` an official Openstack's tool, and Janitor's code is wrapping Openstack python client libraries: -Linux tool to help clean-up tasks for Openstack. +* [python-openstackclient](https://pypi.org/project/python-openstackclient/) to manage authentication and as general API +* [python-neutronclient](https://github.com/openstack/python-neutronclient) to manage network +* [python-novaclient](https://github.com/openstack/python-novaclient) to manage compute nodes and vms +* [python-glanceclient](https://github.com/openstack/python-glanceclient) to manage cloud images +* [python-cinderclient](https://github.com/openstack/python-cinderclient) to manage volumes and block storage -USAGE +## Table of content +* [Motivation](README.md#motivation) +* [Install and usage](README.md#install-and-usage) +* [Setup your dev environment](README.md#local-dev-environment) +* [Running janitor functional tests](tests/README.md#janitor-tests) +* [Contributing](README.md#contributing) +* [Build RPM and release](README.md#build-rpm-and-release) +* [DEMO](README.md#demo) -clean up virtual machines and release floating ips for Openstack keep items declared in the whitelist.txt file: +## Motivation -1. load your Openstack rc to system environment variable or pass as parameter -2. run janitor +I have been using OpenStack continuously since 2014 as a resource to run workloads with the main purpose of +testing software. Manage a healthy Openstack tenant quota is vital when working with CI/CD and a large number of +automated pipelines which run 24/7. A long time ago I and my team were having a lot of headaches with one of internal +Openstack the cluster which at that time had a very limited quota and workload capacity but was important for the +test workflow. With the constant false-positive for test failing due to lack of public/floating IP, storage/volume, +and even virtual CPU when provisioning new VMs, we got the idea of implementing Janitor, a super simple script/codes +to `automate the clean-up task of deleting left-over virtual machines as well network and volume from an Openstack +tenant, all managed by API and remote calls.` - source ~/mytenant-openrc.sh - janitor openstack --whitelist /tmp/whitelist.txt --keystone v3 - janitor openstack --openrc /tmp/mytenant-openrc.sh --whitelist /tmp/whitelist.txt --keystone v3 +## Install and Usage + +For [Fedora](https://fedoraproject.org/): + +1. from [Copr](https://copr.fedorainfracloud.org/coprs/eduardocerqueira/janitor/) + +``` +dnf copr enable eduardocerqueira/janitor +dnf install janitor +``` + +or if you prefer you can install the repo from https://copr.fedorainfracloud.org/coprs/eduardocerqueira/janitor/ +and maybe is needed to disable **gpgcheck=0** + +``` +[eduardocerqueira-janitor] +name=Copr repo for janitor owned by eduardocerqueira +baseurl=https://copr-be.cloud.fedoraproject.org/results/eduardocerqueira/janitor/epel-7-$basearch/ +type=rpm-md +skip_if_unavailable=True +gpgcheck=0 +gpgkey=https://copr-be.cloud.fedoraproject.org/results/eduardocerqueira/janitor/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +enabled_metadata=1 +``` + +2. from local build:: + +``` +make rpm +dnf install rpmbuild/RPMS/x86_64/janitor-0.2-0.x86_64.rpm +``` + +for CentOS: + +**requires openstack repo**. check the latest repo https://wiki.openstack.org/wiki/Release_Naming + +``` +sudo yum install centos-release-openstack-newton.noarch +sudo yum install install rpmbuild/RPMS/x86_64/janitor-0.2-0.x86_64.rpm +``` + +using Janitor to clean up left-over virtual machines and release floating ips for Openstack tenant: + +``` +source ~/mytenant-openrc.sh +janitor openstack --whitelist /tmp/whitelist.txt --keystone v3 + +# or passing the path for your openrc file +janitor openstack --openrc /tmp/mytenant-openrc.sh --whitelist /tmp/whitelist.txt --keystone v3 +``` listing history for your janitor: - janitor history +``` +janitor history +``` + +## local dev environment + +**Requirements:** +* Linux OS *tested on Fedora and CentOS* +* Python3 +* packages: + +``` +# install packages needed to interact with make, build RPM, make release to Copr, and generate doc +sudo dnf install redhat-rpm-config rpm-build python3-devel gcc python3-devel python3-pip python3-wheel python3-setuptools, python3-sphinx copr-cli + +# prep your python env +git clone git@github.com:eduardocerqueira/janitor.git +cd janitor +python3 -m venv venv +source venv/bin/activate +pip install -r requirements/devel.txt + +# check janitor installation +python janitor/cli.py --help +openstack --openrc /home/ecerquei/osp/janitor-openrc.sh --whitelist /home/ecerquei/osp/janitor-whitelist.txt --keystone v3 +``` + +see [running janitor functional tests](tests/README.md#janitor-tests) + +also you can explore the **make tasks** running `make`. + +when running `make doc` the generated doc can be access at : file:///home/user/git/janitor/docs/build/html/index.html + + +## Contributing + +Any idea, suggestions and pacthes are welcome to this project! Fork the project, make your code change, run the test locally +to ensure your changes are not breaking any functionality, remember to run the code static analysis before submitting your +PR and if needed open [issues or discussion](https://github.com/eduardocerqueira/janitor/issues) on this project. + +``` +# make flake8 +``` + +## Build RPM and release + + $ make + + Usage: make where is one of + + clean clean temp files from local workspace + doc generate sphinx documentation and man pages + test run functional/unit tests locally + tarball generate tarball of project + rpm build source codes and generate rpm file + srpm generate SRPM file + build generate srpm and send to build in copr + all clean test doc rpm + flake8 check Python style based on flake8 + +Running from your local machine, you can generate your own RPM running: + + $ make rpm + +if your environment is properly setup you should have your RPM at: /home/user/git/janitor/rpmbuild/RPMS/x86_64/janitor-0.0.2-0.x86_64.rpm + +janitor is being built on [Fedora Copr https://copr.fedorainfracloud.org/coprs/eduardocerqueira/janitor/builds/](https://copr.fedorainfracloud.org/coprs/eduardocerqueira/janitor/builds/) + +running a new build you need to check your ~/.config/copr-fedora file and run: + + make build + +Before starting the release process, check your account permissions in Copr. + + $ make srpm + + 1. copy rpmbuild/SRPMS/janitor-0.0.2-0.src.rpm to janitor/copr + 2. push janitor/copr to github + + `copr-cli` will be used, installed by `sudo yum/dnf install copr-cli` and configure it. + +Request as `Builder` for projects `janitor`, wait until admin approves. + +$ copr-cli build janitor https://github.com/eduardocerqueira/janitor/raw/master/copr/janitor-0.0.2-0.src.rpm + +Go and grab a cup of tea or coffee, the release build will be come out soon :: + + # tag based builds: `https://copr.fedorainfracloud.org/coprs/eduardocerqueira/janitor/builds/` + +**Copr and RPM release helpful Links** +* https://fedorahosted.org/copr/wiki/HowToEnableRepo +* http://fedoraproject.org/wiki/Infrastructure/fedorapeople.org#Accessing_Your_fedorapeople.org_Space +* https://fedorahosted.org/copr/wiki/UserDocs#CanIgiveaccesstomyrepotomyteammate +* https://copr.fedoraproject.org/api/ + -DEMO ----- +## DEMO Running the program without parameters: @@ -52,7 +207,7 @@ Running the program without parameters: Running the program with with parameters to make clean-up: - [ecerquei@dev ~]$ janitor openstack --openrc /home/ecerquei/git/janitor/tests/test-openrc.sh --whitelist /home/ecerquei/git/janitor/tests/whitelist.txt --keystone v2 + [ecerquei@dev ~]$ janitor openstack --openrc /home/ecerquei/git/janitor/tests/test-openrc.sh --whitelist /home/ecerquei/git/janitor/tests/whitelist.txt --keystone v3 +---------------------+--------------+------------------+-----------------------------+--------------------------------------------+--------+----------------------+ | TIMESTAMP | ACTION | NAME | IPs | IMAGE | FLAVOR | CREATED AT (UTC) | +---------------------+--------------+------------------+-----------------------------+--------------------------------------------+--------+----------------------+ @@ -102,99 +257,8 @@ Running the program with parameter to print history file: | 2017-07-13 15:00:57 | deleted | 358be8d1-6d4a-4db7-973f-8369d4ff86f7 | +---------------------+---------+--------------------------------------+ -# For developer and contributers -Note: - At the moment janitor requires python2.7 and it is not compatible with python3 yet! - -This section describes how to build a new RPM for janitor; - -* Fedora 31 - - ``` - sudo dnf install redhat-rpm-config python-devel gcc python2-devel python-pip python-wheel - virtualenv -p /usr/bin/python2.7 venv - source venv/bin/activate - pip install -r requirements/devel.txt - - # test - python janitor/cli.py --help - ``` - -or using **make** so it requires basic packages in your machine I recommend: python-setuptools, python-sphinx, python-devel and gcc - -## RPM / Build - - $ make - - Usage: make where is one of - - clean clean temp files from local workspace - doc generate sphinx documentation and man pages - test run unit tests locally - tarball generate tarball of project - rpm build source codes and generate rpm file - srpm generate SRPM file - build generate srpm and send to build in copr - all clean test doc rpm - flake8 check Python style based on flake8 - -Running from your local machine, you can generate your own RPM running: - - $ make rpm - -and if your environment is setup properly you should have your RPM at: /home/user/git/janitor/rpmbuild/RPMS/x86_64/janitor-0.0.1-1.x86_64.rpm - -janitor is being built on Fedora Copr: https://copr.fedorainfracloud.org/coprs/eduardocerqueira/janitor/builds/ - -running a new build you need to check your ~/.config/copr-fedora file and run: - - make build - - -## install - -Installing from your local machine, after you build your own RPM just run: - -for Fedora: - - sudo dnf install /home/user/git/janitor/rpmbuild/RPMS/x86_64/janitor-0.0.1-1.x86_64.rpm - -for CentOS: - - *requires openstack repo*. you can consult here the latest repo https://wiki.openstack.org/wiki/Release_Naming - - sudo yum install centos-release-openstack-newton.noarch - sudo yum install /home/user/git/janitor/rpmbuild/RPMS/x86_64/janitor-0.0.1-1.x86_64.rpm - -To install from latest RPM: - -**repo:** https://copr.fedorainfracloud.org/coprs/eduardocerqueira/janitor/ - -for Fedora: - - sudo dnf copr enable eduardocerqueira/janitor - -or if needed get the repo from https://copr.fedorainfracloud.org/coprs/eduardocerqueira/janitor/ -and maybe is needed to disable **gpgcheck=0** - - - [eduardocerqueira-janitor] - name=Copr repo for janitor owned by eduardocerqueira - baseurl=https://copr-be.cloud.fedoraproject.org/results/eduardocerqueira/janitor/epel-7-$basearch/ - type=rpm-md - skip_if_unavailable=True - gpgcheck=0 - gpgkey=https://copr-be.cloud.fedoraproject.org/results/eduardocerqueira/janitor/pubkey.gpg - repo_gpgcheck=0 - enabled=1 - enabled_metadata=1 - - -or if links above don't work go to Copr Janitor project for more details how to proceed from here. - -INSTALLATION FAQ ----------------- +### issues Running on CentOS7 even having EPEL and centos-release-openstack-newton I got this error in one of my CentOS server. @@ -251,23 +315,3 @@ the latest version I got was 2.11 that still don't satisfacts. The trick here was, forcing reinstall requests using pip: pip install requests --upgrade - - -## MORE INFO - -For others topics listed below, please generate the sphinx doc in your local machine running the command: - - $ make doc - -and from a browser access: file:///home/user/git/janitor/docs/build/html/index.html - -* Install: -* Guide: -* Build: -* Development: - - - ## How to contribute - - Feel free to fork and send me pacthes or messages if you think this tool can be helpful for any other scenario. - diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 1c5ee3b..706ca84 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -14,6 +14,9 @@ clean up virtual machines and release floating ips for Openstack keep items decl source ~/mytenant-openrc.sh janitor --openstack --whitelist /tmp/whitelist.txt +or passing openrc as argument: + + janitor openstack --openrc /root/openrc.sh --whitelist /var/lib/jenkins/workspace/janitor/janitor_whitelist.txt --keystone v3 listing history for your janitor: diff --git a/docs/source/logo.jpg b/docs/source/logo.jpg new file mode 100644 index 0000000..638b461 Binary files /dev/null and b/docs/source/logo.jpg differ diff --git a/docs/source/motivation.rst b/docs/source/motivation.rst index 3e1cc0a..36c1df3 100644 --- a/docs/source/motivation.rst +++ b/docs/source/motivation.rst @@ -4,10 +4,12 @@ Motivation ========== -One of my Openstack tenants that I use to run tests for an internal tool has -a very limited quota so all the time I need to stop my tests and automations -and spend few minutes cleaning up public/floating ips, destroying -virtual machines and others. The idea to have janitor or a butler is actually -an old idea and a coworker had implemented it but in a more sophisticated -and fashion way this is a simply version and currently only focusing in -Openstack maybe further I'll extend it to AWS, Openshift and others. +I have been using OpenStack continuously since 2014 as a resource to run workloads with the main purpose of +testing software. Manage a healthy Openstack tenant quota is vital when working with CI/CD and a large number of +automated pipelines which run 24/7. A long time ago I and my team were having a lot of headaches with one of internal +Openstack the cluster which at that time had a very limited quota and workload capacity but was important for the +test workflow. With the constant false-positive for test failing due to lack of public/floating IP, storage/volume, +and even virtual CPU when provisioning new VMs, we got the idea of implementing Janitor, a super simple script/codes +to `automate the clean-up task of deleting left-over virtual machines as well network and volume from an Openstack +tenant, all managed by API and remote calls.` + diff --git a/janitor.1 b/janitor.1 index c520428..cd2a885 100644 --- a/janitor.1 +++ b/janitor.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "JANITOR" "1" "Jul 13, 2017" "0.0.1" "janitor" +.TH "JANITOR" "1" "Apr 30, 2021" "0.0.1" "janitor" .SH NAME janitor \- janitor Documentation . @@ -34,13 +34,14 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] Contents: .SH MOTIVATION .sp -One of my Openstack tenants that I use to run tests for an internal tool has -a very limited quota so all the time I need to stop my tests and automations -and spend few minutes cleaning up public/floating ips, destroying -virtual machines and others. The idea to have janitor or a butler is actually -an old idea and a coworker had implemented it but in a more sophisticated -and fashion way this is a simply version and currently only focusing in -Openstack maybe further I’ll extend it to AWS, Openshift and others. +I have been using OpenStack continuously since 2014 as a resource to run workloads with the main purpose of +testing software. Manage a healthy Openstack tenant quota is vital when working with CI/CD and a large number of +automated pipelines which run 24/7. A long time ago I and my team were having a lot of headaches with one of internal +Openstack the cluster which at that time had a very limited quota and workload capacity but was important for the +test workflow. With the constant false\-positive for test failing due to lack of public/floating IP, storage/volume, +and even virtual CPU when provisioning new VMs, we got the idea of implementing Janitor, a super simple script/codes +to \fIautomate the clean\-up task of deleting left\-over virtual machines as well network and volume from an Openstack +tenant, all managed by API and remote calls.\fP .SH EXAMPLES .sp USAGE: @@ -62,6 +63,13 @@ janitor –openstack –whitelist /tmp/whitelist.txt .UNINDENT .UNINDENT .sp +or passing openrc as argument: +.INDENT 0.0 +.INDENT 3.5 +janitor openstack –openrc /root/openrc.sh –whitelist /var/lib/jenkins/workspace/janitor/janitor_whitelist.txt –keystone v3 +.UNINDENT +.UNINDENT +.sp listing history for your janitor: .INDENT 0.0 .INDENT 3.5 diff --git a/janitor.spec.in b/janitor.spec.in index d9e8cb1..1814ebc 100644 --- a/janitor.spec.in +++ b/janitor.spec.in @@ -6,9 +6,9 @@ Group: Applications/Tools License: GPL3+ Source0: http://github.com/eduardocerqueira/janitor/%{name}-%{version}.tar.gz -BuildRequires: python-setuptools -Requires: python -Requires: python-pip +BuildRequires: python3-setuptools +Requires: python3 +Requires: python3-pip Requires: python-click Requires: python-prettytable Requires: python-openstackclient @@ -19,27 +19,29 @@ Requires: python-openstackclient %define _missing_doc_files_terminate_build 0 %description -janitor is a Linux helper tool to perform clean-up for Openstack based on whitelist +janitor is a Linux helper tool to perform clean-up tasks to Openstack based on whitelist %prep %setup -q -n %{name} %build -%{__python} setup.py build +%{__python3} setup.py build %install -%{__python} setup.py install -O1 --skip-build --root %{buildroot} +%{__python3} setup.py install -O1 --skip-build --root %{buildroot} mkdir -p %{buildroot}/%{_mandir}/man1 cp janitor.1 %{buildroot}/%{_mandir}/man1/janitor.1 %files %defattr(755,root,root,755) -%{python_sitelib}/janitor* +%{python3_sitelib}/janitor* %attr (755,root,root)/usr/bin/janitor %doc README.md %doc AUTHORS %{_mandir}/man1/janitor.1.gz %changelog +* Fri Apr 30 2021 Eduardo Cerqueira - 0.0.2 +- code refactoring migrating to python3 * Sun Oct 30 2016 Eduardo Cerqueira - 0.0.1 - initial build diff --git a/janitor/cli.py b/janitor/cli.py index 260825e..e8e75f4 100644 --- a/janitor/cli.py +++ b/janitor/cli.py @@ -43,7 +43,7 @@ def openstack(openrc, whitelist, keystone): vms = openstack.get_all_instances() volumes = openstack.get_volume(status={'status': 'available'}) except Exception as ex: - print ex + print(ex) exit(1) # clean up clean = Clean(vms, whitelist, volumes, openstack) diff --git a/janitor/module/clean.py b/janitor/module/clean.py index a1ea786..87668e6 100644 --- a/janitor/module/clean.py +++ b/janitor/module/clean.py @@ -15,8 +15,8 @@ from fnmatch import fnmatch from os.path import exists -from util import file_mgmt -from history import History +from .util import file_mgmt +from .history import History class Clean(object): @@ -32,7 +32,7 @@ def __init__(self, vms, wlist_path, volumes, driver): def read_whitelist(self): """ """ if not exists(self.wlist_path): - raise Exception("File %s not found" % self.wlist_path) + raise Exception(f"File {self.wlist_path} not found") wlist = file_mgmt('r', file_path=self.wlist_path) return wlist.split() @@ -52,27 +52,36 @@ def run(self): deleted = [] for vm in self.vm_list: if vm['name'] not in to_keep_names: - if self.driver.delete_instance(vm): - deleted.append(vm) - else: - print "Could not delete vm %s" % vm['name'] + try: + if self.driver.delete_instance(vm): + deleted.append(vm) + else: + print(f"Could not delete vm {vm['name']}") + except Exception as ex: + print(ex) # delete zoombies floating ips ips_deleted = [] - zoombies_ips = self.driver.get_zoombies_floating_ips() - for ip in zoombies_ips: - if self.driver.delete_floating_ip(ip['id']): - ips_deleted.append(ip) - else: - print "Could not delete ip %s" % ip + zombie_ips = self.driver.get_zoombies_floating_ips() + try: + for ip in zombie_ips: + if self.driver.delete_floating_ip(ip['id']): + ips_deleted.append(ip) + else: + print(f"Could not delete ip {ip}") + except Exception as ex: + print(ex) # delete volumes vols_deleted = [] for vol in self.volumes: # extra verification for not attached volumes - if len(vol.attachments) == 0: - self.driver.cinder.volumes.delete(vol.id) - vols_deleted.append(vol.id) + try: + if len(vol.attachments) == 0: + self.driver.cinder.volumes.delete(vol.id) + vols_deleted.append(vol.id) + except Exception as ex: + print(ex) # history history = History(to_keep, deleted, ips_deleted, vols_deleted) diff --git a/janitor/module/history.py b/janitor/module/history.py index 322525e..9249b3d 100644 --- a/janitor/module/history.py +++ b/janitor/module/history.py @@ -15,7 +15,7 @@ from prettytable import PrettyTable from datetime import datetime -from util import file_mgmt +from .util import file_mgmt from os import getenv from os.path import join, exists @@ -38,14 +38,14 @@ def print_history(self): """print history file""" if exists(self.history_path): history = file_mgmt('r', file_path=self.history_path) - print history + print(history) else: - print "no history yet" + print("no history yet") def print_report(self): """print current report""" if self.report: - print self.report + print(self.report) def create_report(self): """ @@ -57,17 +57,17 @@ def create_report(self): # instances/vm report vm_report = PrettyTable(['TIMESTAMP', 'ACTION', 'NAME', 'IPs', - 'IMAGE', 'FLAVOR', 'CREATED AT (UTC)']) + 'IMAGE', 'CREATED AT (UTC)']) now = datetime.now().strftime('%Y-%m-%d %H:%M:%S') for vm in self.deleted: vm_report.add_row([now, 'deleted', vm['name'], vm['ips'], vm['image'], - vm['flavor'], vm['created_at']]) + vm['created_at']]) for vm in self.keep: vm_report.add_row([now, 'in whitelist', vm['name'], vm['ips'], vm['image'], - vm['flavor'], vm['created_at']]) + vm['created_at']]) # floating ip report ip_report = PrettyTable(['TIMESTAMP', 'ACTION', 'FLOATING IP']) @@ -86,13 +86,13 @@ def create_report(self): # group all reports full_report = None - if vm_report._rows.__len__() > 0: + if vm_report.rows.__len__() > 0: full_report = str(vm_report) + "\n" - if ip_report._rows.__len__() > 0: + if ip_report.rows.__len__() > 0: full_report = full_report + str(ip_report) + "\n\n" - if vol_report._rows.__len__() > 0: + if vol_report.rows.__len__() > 0: full_report = full_report + str(vol_report) + "\n\n" return full_report diff --git a/janitor/provider/openstack.py b/janitor/provider/openstack.py index 85ae1cf..7c4fd46 100644 --- a/janitor/provider/openstack.py +++ b/janitor/provider/openstack.py @@ -144,7 +144,7 @@ def get_all_instances(self): os_img = self.glance.images.get(vm.image['id']) image = os_img['name'] except Exception as ex: - print ex + print(ex) image = "NA, prev deleted" if vm.name and vm.name is not None: @@ -170,9 +170,9 @@ def get_all_instances(self): vm_list.append(instance) except Exception as ex: # for troubleshooting - print ex + print(ex) if vm: - print "WARN: check this vm %s" % vm + print(f"WARN: check this vm {vm}") raise (ex) return vm_list @@ -182,14 +182,15 @@ def delete_instance(self, vm): self.nova.servers.delete(vm['obj']) return True except Exception as ex: - print ex + print(ex) def get_server_ips(self, server): """get IPs for a given server object""" server_ips = [] - for key, value in server.addresses.iteritems(): - for elem in value: - server_ips.append(elem['addr']) + if server.addresses: + for key, value in server.addresses.items(): + for elem in value: + server_ips.append(elem['addr']) return server_ips def get_zoombies_floating_ips(self): @@ -207,11 +208,11 @@ def delete_floating_ip(self, ip): self.neutron.delete_floatingip(ip) return True except Exception as ex: - print ex + print(ex) def get_volume(self, status): """query cinder for all available volume""" try: return self.cinder.volumes.list(search_opts=status) except Exception as ex: - print ex + print(ex) diff --git a/requirements/devel.txt b/requirements/devel.txt index 6394997..80b2c46 100644 --- a/requirements/devel.txt +++ b/requirements/devel.txt @@ -1,10 +1,12 @@ -r production.txt -flake8==2.6.2 +flake8 coverage>=3.7.1 mock>=1.0.1 setuptools -# docs +# docs and static code analysis Sphinx pep8 +mccabe>=0.6,<0.7 # version needed by pylint pylint -pytest \ No newline at end of file +pytest +nose \ No newline at end of file diff --git a/requirements/production.txt b/requirements/production.txt index 23e2f29..c0903fc 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,5 +1,5 @@ click -PrettyTable +PrettyTable<0.8,>=0.7.1 # version required by python-glanceclient==3.3.0 python-openstackclient python-neutronclient python-novaclient diff --git a/setup.py b/setup.py index 4d9b6ae..b21897c 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ def get_install_requires(): requires.append(line) return requires, links + install_requires, dependency_links = get_install_requires() setup( diff --git a/tests/README.md b/tests/README.md index d986e4e..f758bcb 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,45 +1,53 @@ -RUNNING TESTS -============= +# Janitor Tests -Openstack tests requires you have your openstack tenant openrc.sh file that -provides the Openstack credentials. Janitor's tests reads system -environment variable if you perform a source to your openrc.sh or from a -file that should be placed at janitor/tests/test-openrc.sh. There is a -test-openrc-sample.sh file you can use to start it, just remove -sample or copy -and paste to a new file without -sample and fill out with your credentials. +**Requirements:** +* Openstack cluster properly setup +* Openstack tenant and credentials +* Openstack openrc.sh file +* whitelist.txt file +* python venv with janitor/requirements installed see [local dev environment](../README.md#local-dev-environment) -Also whitelist.txt file is required and should be placed at same place like: -janitor/tests/whitelist.txt +The tests can handle authentication by environment variables or loading the variables from a physical openrc.sh file, +you can use the template `janitor/tests/test-openrc-sample.sh` creating a copy and replacing the content with your +account and Openstack data. -DATA FOR TESTS ---------------- +Or using environment variable to point the location of your openrc.sh file as showing below: +``` +export OPENSTACK_CREDS=/home/user/openrc.sh +``` -running the openstack nova cli below you can generate content or vms for your -tests: +`whitelist.txt` file is required and should be placed in the same folder: `janitor/tests/whitelist.txt` which will +contain the name of virtual machines to be kept and not deleted when Janitor runs. - nova boot --flavor 2 --key-name mykeyname --image e636fa2e-a1fc-48c0-97c4-04fc74651281 test2 +## Generating data for test +It is an optional step, feel free to skip it if you have data in your Openstack for testing. -HOW TO RUN ALL TESTS --------------------- +Running the commands below you can generate content or vms for your tests, replace values from `--key-name --image --network` +with correspondent from your Openstack: -you have to install janitor from branch or version you want to run the tests. -Go to janitor folder and run: +``` +openstack server create --key-name MyKeyName --flavor m1.small --image FEDORA-34-x86_64-latest --network d655dcd0-b593-439c-997b-aa5bc8c03a3a Fedora34 +openstack server create --key-name MyKeyName --flavor m1.small --image FEDORA-33-x86_64-latest --network d655dcd0-b593-439c-997b-aa5bc8c03a3a Fedora33 +openstack server create --key-name MyKeyName --flavor m1.small --image FEDORA-32-x86_64-latest --network d655dcd0-b593-439c-997b-aa5bc8c03a3a Fedora32 +``` + +## Running tests - python -m pytest tests -v +``` +cd tests +# running all tests from a file +pytest test_openstack_credentials.py -v -HOW TO RUN AN INDIVIDUAL TEST ------------------------------ +# running specific test +pytest test_openstack_instances.py -v -k test_get_ip_all_instances -you have to install janitor from branch or version you want to run the tests. -Go to janitor folder and run: - - python -m pytest tests/openstack/test_instances.py -v - - -ISSUES ------- +# running all tests +pytest -v +``` + +## Issues 1. classpath @@ -49,9 +57,3 @@ ISSUES PYTHONPATH=janitor pytest tests -v on this example I am running the command above from my janitor folder. - - -REFERENCE LINKS ----------------- - -http://www.tilcode.com/pytest-cli-tips-tricks-settings/ diff --git a/tests/helper.py b/tests/helper.py index 5932fc4..080bbd4 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -16,32 +16,31 @@ from os import getenv, getcwd from os.path import exists, join, dirname - ROOT = dirname(getcwd()) def get_openstack_creds(): - ops_creds = getenv("OPENSTACK_CREDS") if ops_creds: - print "Using %s" % ops_creds + print(f"Using {ops_creds}") return ops_creds # root and workspace folder elif exists(join(ROOT, 'test-openrc.sh')): ops_creds = join(ROOT, 'test-openrc.sh') - print "Using %s" % ops_creds + print(f"Using {ops_creds}") return ops_creds # tests folder elif exists(join(ROOT, 'tests/test-openrc.sh')): ops_creds = join(ROOT, 'tests/test-openrc.sh') - print "Using %s" % ops_creds + print(f"Using {ops_creds}") return ops_creds # janitor folder elif exists(join(ROOT, 'janitor/tests/test-openrc.sh')): ops_creds = join(ROOT, 'janitor/tests/test-openrc.sh') - print "Using %s" % ops_creds + print(f"Using {ops_creds}") return ops_creds else: raise Exception("missing Openstack credentials") + OPENSTACK_CREDS_PATH = get_openstack_creds() diff --git a/tests/test_openstack_credentials.py b/tests/test_openstack_credentials.py index d8c23ae..65daf41 100644 --- a/tests/test_openstack_credentials.py +++ b/tests/test_openstack_credentials.py @@ -14,8 +14,8 @@ # import pytest -from provider.openstack import OpenstackSDK -from helper import OPENSTACK_CREDS_PATH +from janitor.provider.openstack import OpenstackSDK +from .helper import OPENSTACK_CREDS_PATH class TestCredentials(object): @@ -23,7 +23,7 @@ class TestCredentials(object): def test_negative_credentials(self): try: OpenstackSDK() - except: + except Exception: assert True def test_credentials(self): diff --git a/tests/test_openstack_instances.py b/tests/test_openstack_instances.py index abaecac..c38876f 100644 --- a/tests/test_openstack_instances.py +++ b/tests/test_openstack_instances.py @@ -14,7 +14,7 @@ # import pytest -from provider.openstack import OpenstackSDK +from janitor.provider.openstack import OpenstackSDK from helper import OPENSTACK_CREDS_PATH @@ -30,17 +30,24 @@ def test_get_all_instances(self): def test_get_ip_all_instances(self): openstack = OpenstackSDK(OPENSTACK_CREDS_PATH) servers = openstack.get_all_instances() + vm_no_ip = [] for server in servers: - if openstack.get_server_ips(server['obj']): - assert True - else: + try: + if openstack.get_server_ips(server['obj']): + pass + else: + vm_no_ip.append(server['obj']) + except Exception: assert False + print(f"Found {len(vm_no_ip)} vms with no assigned IP out of {len(servers)}") + assert True + def test_get_zoombies_floating_ips(self): openstack = OpenstackSDK(OPENSTACK_CREDS_PATH) ips_zoombies = openstack.get_zoombies_floating_ips() if ips_zoombies: assert True else: - print "ZERO, it is fine!" + print("ZERO, it is fine!") assert True diff --git a/tests/test_openstacksdk.py b/tests/test_openstacksdk.py index 5fcf73a..0f41e09 100644 --- a/tests/test_openstacksdk.py +++ b/tests/test_openstacksdk.py @@ -15,7 +15,7 @@ import pytest -from provider.openstack import OpenstackSDK +from janitor.provider.openstack import OpenstackSDK from helper import OPENSTACK_CREDS_PATH diff --git a/tests/whitelist.txt b/tests/whitelist.txt index 48ab2c3..9553ccc 100644 --- a/tests/whitelist.txt +++ b/tests/whitelist.txt @@ -1,4 +1,24 @@ centos rhel jslave* -windows* \ No newline at end of file +windows* +*jeff* +*ecerquei* +*esakaiev* +*pakotvan* +*dlezzoum* +*jenkins* +jslave* +*insights* +*testday* +*fstavela* +*msager* +*opacut* +psav* +jaudet* +*vbelchio* +pvala* +*mlahane* +*uenkhbay* +*dgaikwad* +*digitronik* \ No newline at end of file diff --git a/tox.ini b/tox.ini index 29196b8..a88ab5c 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ skipsdist = True deps = -r{toxinidir}/requirements/devel.txt setenv = PYTHONPATH = {toxinidir}:{toxinidir} -commands = python driver.py --help +commands = python cli.py --help [run] branch = True