Merge pull request #953 from zenhack/obmd-integration-extract-info

obmd data migration script

.travis.yml

@@ -17,6 +17,7 @@ addons:
     packages:
       - apache2
       - libapache2-mod-wsgi
+      - golang
 
 branches:
   except:
@@ -39,4 +40,5 @@ script:
   - sh -e ci/run_unit_tests.sh
   - sh -e ci/apache/run_integration_tests.sh
   - sh -e ci/keystone/run_integration_tests.sh
+  - sh -e ci/obmd/run_integration_tests.sh
   - sh -e ci/deployment-mock-networks/run_integration_tests.sh

ci/obmd/run_integration_tests.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+set -ex
+
+py.test tests/integration/migrate_ipmi_info.py

hil/commands/admin.py

@@ -1,12 +1,14 @@
 """Implement the hil-admin command."""
 from hil import config, model
 from hil.commands import db
+from hil.commands.migrate_ipmi_info import MigrateIpmiInfo
 from hil.commands.util import ensure_not_root
 from hil.flaskapp import app
 from flask.ext.script import Manager
 
 manager = Manager(app)
 manager.add_command('db', db.command)
+manager.add_command('migrate-ipmi-info', MigrateIpmiInfo())
 
 
 def main():

hil/commands/migrate_ipmi_info.py

@@ -0,0 +1,131 @@
+"""Helper commands for moving IPMI info from the database to obmd.
+
+This is part of our obmd migration strategy; see issue #928.
+"""
+
+import sys
+import json
+
+import requests
+
+from flask_script import Command, Option
+
+from hil import server
+from hil.flaskapp import app
+from hil import model
+
+
+class MigrateIpmiInfo(Command):
+    """Migrate ipmi info to obmd"""
+
+    option_list = (
+        Option('--obmd-base-url', dest='obmd_base_url',
+               help='Base url for the obmd api'),
+        Option('--obmd-admin-token', dest='obmd_admin_token',
+               help='Admin token for obmd'),
+    )
+
+    # the correct arguments to this are a function of the available options;
+    # it's normal for subclasses to have implementations with different
+    # arguments.
+    #
+    # pylint: disable=arguments-differ
+    def run(self, obmd_base_url, obmd_admin_token):
+        server.init()
+        with app.app_context():
+            info = db_extract_ipmi_info()
+            obmd_upload_ipmi_info(obmd_base_url, obmd_admin_token, info)
+            db_add_obmd_info(obmd_base_url, obmd_admin_token)
+
+
+def db_extract_ipmi_info():
+    """Extract all nodes' ipmi connection info from the database.
+
+    This returns an dictionary of the form:
+
+        {
+            "node-23": {
+                "host": "172.16.32.23",
+                "user": "admin",
+                "password": "secret"
+            },
+            "node-46": {
+                "host": "172.16.32.46",
+                "user": "admin",
+                "password": "changeme"
+            }
+        }
+
+    It will fail if any of the nodes in the database have obms with a type
+    other than ipmi.
+
+    This must be run inside of an app context.
+    """
+    obms = model.Obm.query.all()
+
+    info = {}
+
+    ipmi_api_name = 'http://schema.massopencloud.org/haas/v0/obm/ipmi'
+
+    for obm in obms:
+        # XXX: apparently we've incorrectly specified the relationship between
+        # Obms and nodes, such that the node attribute returns a list, even
+        # though there can only ever be one node. If we were keeping pluggable
+        # obm support for longer, We'd just fix this, but since we're removing
+        # the functionality soon it's easier to just do this and not have to
+        # worry about what other code we might disturb:
+        node = obm.node[0]
+
+        if obm.type != ipmi_api_name:
+            sys.exit(("Node %s{label} has an obm of unspported "
+                      "type %s{type}").format({
+                          'label': node.label,
+                          'type': obm.type,
+                      }))
+        info[node.label] = {
+            'host': obm.host,
+            'user': obm.user,
+            'password': obm.password,
+        }
+
+    return info
+
+
+def obmd_upload_ipmi_info(obmd_base_url, obmd_admin_token, info):
+    """Upload nodes' info to obmd.
+
+    `info` should be a dictionary of the form returned by
+    `db_extract_ipmi_info`.
+
+    `obmd_base_url` is the base URL for the obmd api. i.e. a node named
+    "example_node" would be at the URL `obmd_base_url + '/node/example_node'.`
+
+    `obmd_admin_token` is the admin token to use when authenticating against
+    obmd.
+    """
+    sess = requests.Session()
+    sess.auth = ('admin', obmd_admin_token)
+
+    for key, val in info.items():
+        sess.put(obmd_base_url + '/node/' + key, data=json.dumps({
+            'type': 'ipmi',
+            'info': {
+                'addr': val['host'],
+                'user': val['user'],
+                'pass': val['password'],
+            },
+        }))
+
+
+def db_add_obmd_info(obmd_base_url, obmd_admin_token):
+    """Add obmd connection info to the HIL database.
+
+    This assumes each node is available from obmd at the URL
+    `obmd_base_url + '/node/' + node.label`.
+
+    This must be run inside of an app context.
+    """
+    model.Node.query.update({'obmd_admin_token': obmd_admin_token})
+    for node in model.Node.query.all():
+        node.obmd_uri = obmd_base_url + '/node/' + node.label
+    model.db.session.commit()

tests/integration/migrate_ipmi_info.py

@@ -0,0 +1,141 @@
+"""Tests for the `hil-admin migrate-ipmi-info` command."""
+
+from subprocess import check_call, Popen
+import shutil
+import tempfile
+import json
+import os
+
+import requests
+import pytest
+
+from hil.test_common import fresh_database
+from hil.flaskapp import app
+from hil import config, model
+
+ADMIN_TOKEN = '01234567890123456789012345678901'
+OBMD_BASE_URL = 'http://localhost:8080'
+
+
+@pytest.fixture()
+def tmpdir():
+    """Create a temporary directory to store various files."""
+    path = tempfile.mkdtemp()
+    cwd = os.getcwd()
+    os.chdir(path)
+    yield path
+    os.chdir(cwd)
+    shutil.rmtree(path)
+
+
+@pytest.fixture()
+def configure(tmpdir):
+    """Set up HIL configuration.
+
+    This creates a hil.cfg in tmpdir, and loads it. The file needs to be
+    written out separately , since we invoke other commands that read it,
+    besides the test process.
+    """
+    cfg = '\n'.join([
+        "[extensions]",
+        "hil.ext.network_allocators.null =",
+        "hil.ext.auth.null =",
+        "hil.ext.obm.ipmi = ",
+        "[devel]",
+        "dry_run = True",
+        "[headnode]",
+        "base_imgs = base-headnode, img1, img2, img3, img4",
+        "[database]",
+        "uri = sqlite:///" + tmpdir + "/hil.db",
+    ])
+    with open(tmpdir + '/hil.cfg', 'w') as f:
+        f.write(cfg)
+    config.load(tmpdir + '/hil.cfg')
+    config.configure_logging()
+    config.load_extensions()
+
+
+fresh_database = pytest.fixture(fresh_database)
+
+
+@pytest.fixture()
+def run_obmd(tmpdir):
+    """Set up and start obmd."""
+    check_call(['go', 'get', 'github.com/CCI-MOC/obmd'])
+
+    config_file_path = tmpdir + '/obmd-config.json'
+
+    with open(config_file_path, 'w') as f:
+        f.write(json.dumps({
+            'AdminToken': ADMIN_TOKEN,
+            'ListenAddr': ':8080',
+        }))
+
+    proc = Popen(['obmd', '-config', config_file_path])
+    try:
+        yield
+    finally:
+        proc.terminate()
+        proc.wait()
+
+
+pytestmark = pytest.mark.usefixtures('configure',
+                                     'fresh_database',
+                                     'run_obmd')
+
+
+def test_obmd_migrate(tmpdir):
+    """The test proper.
+
+    Create some nodes, run the script, and verify that it has done the right
+    thing.
+    """
+    from hil.ext.obm.ipmi import Ipmi
+
+    # Add some objects to the hil database:
+    with app.app_context():
+        for i in range(4):
+            node = model.Node(label='node-%d' % i,
+                              obm=Ipmi(user='admin',
+                                       host='10.0.0.%d' % (100 + i),
+                                       password='changeme'))
+            model.db.session.add(node)
+        model.db.session.commit()
+
+    check_call([
+        'hil-admin', 'migrate-ipmi-info',
+        '--obmd-base-url', OBMD_BASE_URL,
+        '--obmd-admin-token', ADMIN_TOKEN,
+    ])
+
+    with app.app_context():
+        for node in model.Node.query.all():
+
+            # Check that the db info was updated correctly:
+            assert node.obmd_admin_token == ADMIN_TOKEN, (
+                "Node %s{label}'s admin token was incorrect: %s{token}"
+                .format(
+                    label=node.label,
+                    token=node.obmd_admin_token,
+                )
+            )
+            assert node.obmd_uri == OBMD_BASE_URL + '/node/' + node.label, (
+                "Node %s{label}'s obmd_uri was incorrect: %s{uri}"
+                .format(
+                    label=node.label,
+                    uri=node.obmd_uri,
+                )
+            )
+
+            # Make sure obmd thinks the nodes are there; if so it should be
+            # possible to get a token:
+            sess = requests.Session()
+            sess.auth = ('admin', ADMIN_TOKEN)
+            resp = sess.post(node.obmd_uri + '/token')
+            assert resp.ok, (
+                "Failure getting token for node %s{label} from obmd; "
+                "response: %s{resp}".format(
+                    label=node.label,
+                    resp=resp,
+                )
+            )