From 63590c4141bb6d78baabbd92a65e44529911eb2a Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Mon, 14 Mar 2016 20:54:38 +0530 Subject: [PATCH 01/62] Initial structure --- __init__.py | 0 __init__.pyc | Bin 0 -> 155 bytes app.py | 14 ++++++++++++++ controllers/__init__.py | 0 controllers/__init__.pyc | Bin 0 -> 167 bytes controllers/root.py | 22 ++++++++++++++++++++++ controllers/root.pyc | Bin 0 -> 1519 bytes model/__init__.py | 15 +++++++++++++++ model/__init__.pyc | Bin 0 -> 781 bytes tests/__init__.py | 22 ++++++++++++++++++++++ tests/config.py | 25 +++++++++++++++++++++++++ tests/test_functional.py | 22 ++++++++++++++++++++++ tests/test_units.py | 7 +++++++ tests/test_units.pyc | Bin 0 -> 677 bytes 14 files changed, 127 insertions(+) create mode 100644 __init__.py create mode 100644 __init__.pyc create mode 100644 app.py create mode 100644 controllers/__init__.py create mode 100644 controllers/__init__.pyc create mode 100644 controllers/root.py create mode 100644 controllers/root.pyc create mode 100644 model/__init__.py create mode 100644 model/__init__.pyc create mode 100644 tests/__init__.py create mode 100644 tests/config.py create mode 100644 tests/test_functional.py create mode 100644 tests/test_units.py create mode 100644 tests/test_units.pyc diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/__init__.pyc b/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57b1749a7cc0c3cea9839e918549fbf5e8d1efe5 GIT binary patch literal 155 zcmZSn%*&NM?`~K!0~9a-b88Qw%PKxb6jncwyW_R zyctgc_h38i_5v(wT-)aypUE#`&Aj6x%e z0~U@_TDhc3TYAK1+%&Y`)MT5oM0Q zyU)6lgQQeUG`Vq?V!-a4W8GQ~%X!FNP~kqa7E#qkZP6yPRZ-@NWe%biv$S4Im!@Y!6_leeV>|XKWlBj(hb)ZHT)0~!=H<=qXb`fD6DNz(TOl`JQp5C9n zec?3sSlH57u~H@G>0R=0Q?P`)kM8qzYcjNyXl-M=Qf@d9;*^IYRxXA@1@pi(Tp9RQ zKZAe_oD9Ie1Y-=7#O~U_5qUs990%*o+=|?X*FIdaUptV4a1&l?i+1xdF8*-(iiu~V zOzV^it|VXE4NNx^+KUxB$EDV>;%lU|q;qqamMi=uS*&}O=ZI`9$>_cXQY^z4$|0pZ zek|HheEi;J`zg6{n|mNPdA=E%hh|`QO}Ndr4px%XX@yCm$s|eD9O&(c-5pZWB=LzYkF?q)9(Gp|I~g4|aobsC_G5olQlmT}|@kIKx>p8{v7@ fl|arlvt$0$X{@^jtdzd#NPH)w<%F~yGxYxfc@I^i literal 0 HcmV?d00001 diff --git a/model/__init__.py b/model/__init__.py new file mode 100644 index 0000000..d983f7b --- /dev/null +++ b/model/__init__.py @@ -0,0 +1,15 @@ +from pecan import conf # noqa + + +def init_model(): + """ + This is a stub method which is called at application startup time. + + If you need to bind to a parsed database configuration, set up tables or + ORM classes, or perform any database initialization, this is the + recommended place to do it. + + For more information working with databases, and some common recipes, + see http://pecan.readthedocs.org/en/latest/databases.html + """ + pass diff --git a/model/__init__.pyc b/model/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..688a7c371a5e862dcbb88b6d6406b8973d3f7a3f GIT binary patch literal 781 zcmcJN&x#Z=5XLhz3Nnk}3kY=_*ky8GLU9l z_yAhTjw^UKB$e(j>0f=R&f@3u<>C8}&qkfwT;?}2>Prb$`b>GH!c5)I)XdJ+SEcUf z$}ZIH>WS=dzoe~Z^2F`cq_}%THTA8G+E#>fyK3J^Jy}2o%-K7G>{67iSa{07F3@_}Iq!FHN)tqq-JD?SVxJK*mMWS>#w}KY=FaV> z(n(iPiJ_5j>#0I29rkJS5sb%Lsa}Z&naVDe02ohyK`uIW=-rnyiO(nDkvgQ1Ng+_Q zl0ADg)O^_lj?b3g2|1)f;9{dc9POCO!Nols9gmN>#0AQEY6{JN2vP!}yDav4s!9YS zbH3I(lfk$ygjRgmWa>JVy{4!=GF8@p>QL7s2j9QGS;-$!|AhKMG|eIF?DDXkoVPP(jC2y;4rPY@i%>nmxsY9~l^)R#=4Z}L0cqzF4)|c94sO=)Ci<{L#CVncR cd*3v5)2_{bcJxeAyb?E)r`IiA_w!=$8>t50h5!Hn literal 0 HcmV?d00001 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..78ea527 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,22 @@ +import os +from unittest import TestCase +from pecan import set_config +from pecan.testing import load_test_app + +__all__ = ['FunctionalTest'] + + +class FunctionalTest(TestCase): + """ + Used for functional tests where you need to test your + literal application and its integration with the framework. + """ + + def setUp(self): + self.app = load_test_app(os.path.join( + os.path.dirname(__file__), + 'config.py' + )) + + def tearDown(self): + set_config({}, overwrite=True) diff --git a/tests/config.py b/tests/config.py new file mode 100644 index 0000000..4d41a1d --- /dev/null +++ b/tests/config.py @@ -0,0 +1,25 @@ +# Server Specific Configurations +server = { + 'port': '8080', + 'host': '0.0.0.0' +} + +# Pecan Application Configurations +app = { + 'root': 'dockyard.controllers.root.RootController', + 'modules': ['dockyard'], + 'static_root': '%(confdir)s/../../public', + 'template_path': '%(confdir)s/../templates', + 'debug': True, + 'errors': { + '404': '/error/404', + '__force_dict__': True + } +} + +# Custom Configurations must be in Python dictionary format:: +# +# foo = {'bar':'baz'} +# +# All configurations are accessible at:: +# pecan.conf diff --git a/tests/test_functional.py b/tests/test_functional.py new file mode 100644 index 0000000..b33259c --- /dev/null +++ b/tests/test_functional.py @@ -0,0 +1,22 @@ +from unittest import TestCase +from webtest import TestApp +from dockyard.tests import FunctionalTest + + +class TestRootController(FunctionalTest): + + def test_get(self): + response = self.app.get('/') + assert response.status_int == 200 + + def test_search(self): + response = self.app.post('/', params={'q': 'RestController'}) + assert response.status_int == 302 + assert response.headers['Location'] == ( + 'http://pecan.readthedocs.org/en/latest/search.html' + '?q=RestController' + ) + + def test_get_not_found(self): + response = self.app.get('/a/bogus/url', expect_errors=True) + assert response.status_int == 404 diff --git a/tests/test_units.py b/tests/test_units.py new file mode 100644 index 0000000..573fb68 --- /dev/null +++ b/tests/test_units.py @@ -0,0 +1,7 @@ +from unittest import TestCase + + +class TestUnits(TestCase): + + def test_units(self): + assert 5 * 5 == 25 diff --git a/tests/test_units.pyc b/tests/test_units.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ba4ac8870a7a3fc345f1414e8049cfdf3cacdac GIT binary patch literal 677 zcmcIgOHRWu5FIC_f=VD@fy50WDv=8y1XS!$7bzgojg=-2vYMYV?t(3F4sOONH~_qH zB6@;ap6Bs+JoDa!KZld`*Y|Tq!wPZU!1w}?V=F3&Dn(00Di6pgDiq}*eG+{Ty(|OD zN3@9ckZymmA_dRw10D^#S<1#8440?cI{xn@d3P{A0T_}7=!dc>nJj#nOmf5m*bK&J z05k`YbwGSdj_BS0TSOx~&8_Eg4%Z6wWRBq7GMmoXroQjGrt9PKgH2gPILQM)z>8hg z#;tAjYGcb>cTMBs)?{ftAZKY&G-b~53hLr%n`Z0u)l6dt!R%3QMCxVx<|Zg8@jf!f zjYSNpNG9#)q;;KEMr)s|b=BmX64ylNZK>^sUn0}>a}ZYJV7{d#5`=kz2l9ElCgZ1Z N)$6|&Vt!P|;UB4uoPz)W literal 0 HcmV?d00001 From a0a4a1a94f1f72b1127fd54091a72a1dc4687013 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Mon, 14 Mar 2016 20:58:53 +0530 Subject: [PATCH 02/62] Removed .pyc file --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc From 7a1ce10f1c2da46f91368ae41515939e3a090e50 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Mon, 14 Mar 2016 21:03:11 +0530 Subject: [PATCH 03/62] Removed pyc files --- __init__.pyc | Bin 155 -> 0 bytes controllers/__init__.pyc | Bin 167 -> 0 bytes controllers/root.pyc | Bin 1519 -> 0 bytes model/__init__.pyc | Bin 781 -> 0 bytes tests/test_units.pyc | Bin 677 -> 0 bytes 5 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 __init__.pyc delete mode 100644 controllers/__init__.pyc delete mode 100644 controllers/root.pyc delete mode 100644 model/__init__.pyc delete mode 100644 tests/test_units.pyc diff --git a/__init__.pyc b/__init__.pyc deleted file mode 100644 index 57b1749a7cc0c3cea9839e918549fbf5e8d1efe5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 155 zcmZSn%*&NM?`~K!0~9a-b88Qw%PKxb6jncwyW_R zyctgc_h38i_5v(wT-)aypUE#`&Aj6x%e z0~U@_TDhc3TYAK1+%&Y`)MT5oM0Q zyU)6lgQQeUG`Vq?V!-a4W8GQ~%X!FNP~kqa7E#qkZP6yPRZ-@NWe%biv$S4Im!@Y!6_leeV>|XKWlBj(hb)ZHT)0~!=H<=qXb`fD6DNz(TOl`JQp5C9n zec?3sSlH57u~H@G>0R=0Q?P`)kM8qzYcjNyXl-M=Qf@d9;*^IYRxXA@1@pi(Tp9RQ zKZAe_oD9Ie1Y-=7#O~U_5qUs990%*o+=|?X*FIdaUptV4a1&l?i+1xdF8*-(iiu~V zOzV^it|VXE4NNx^+KUxB$EDV>;%lU|q;qqamMi=uS*&}O=ZI`9$>_cXQY^z4$|0pZ zek|HheEi;J`zg6{n|mNPdA=E%hh|`QO}Ndr4px%XX@yCm$s|eD9O&(c-5pZWB=LzYkF?q)9(Gp|I~g4|aobsC_G5olQlmT}|@kIKx>p8{v7@ fl|arlvt$0$X{@^jtdzd#NPH)w<%F~yGxYxfc@I^i diff --git a/model/__init__.pyc b/model/__init__.pyc deleted file mode 100644 index 688a7c371a5e862dcbb88b6d6406b8973d3f7a3f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 781 zcmcJN&x#Z=5XLhz3Nnk}3kY=_*ky8GLU9l z_yAhTjw^UKB$e(j>0f=R&f@3u<>C8}&qkfwT;?}2>Prb$`b>GH!c5)I)XdJ+SEcUf z$}ZIH>WS=dzoe~Z^2F`cq_}%THTA8G+E#>fyK3J^Jy}2o%-K7G>{67iSa{07F3@_}Iq!FHN)tqq-JD?SVxJK*mMWS>#w}KY=FaV> z(n(iPiJ_5j>#0I29rkJS5sb%Lsa}Z&naVDe02ohyK`uIW=-rnyiO(nDkvgQ1Ng+_Q zl0ADg)O^_lj?b3g2|1)f;9{dc9POCO!Nols9gmN>#0AQEY6{JN2vP!}yDav4s!9YS zbH3I(lfk$ygjRgmWa>JVy{4!=GF8@p>QL7s2j9QGS;-$!|AhKMG|eIF?DDXkoVPP(jC2y;4rPY@i%>nmxsY9~l^)R#=4Z}L0cqzF4)|c94sO=)Ci<{L#CVncR cd*3v5)2_{bcJxeAyb?E)r`IiA_w!=$8>t50h5!Hn diff --git a/tests/test_units.pyc b/tests/test_units.pyc deleted file mode 100644 index 4ba4ac8870a7a3fc345f1414e8049cfdf3cacdac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 677 zcmcIgOHRWu5FIC_f=VD@fy50WDv=8y1XS!$7bzgojg=-2vYMYV?t(3F4sOONH~_qH zB6@;ap6Bs+JoDa!KZld`*Y|Tq!wPZU!1w}?V=F3&Dn(00Di6pgDiq}*eG+{Ty(|OD zN3@9ckZymmA_dRw10D^#S<1#8440?cI{xn@d3P{A0T_}7=!dc>nJj#nOmf5m*bK&J z05k`YbwGSdj_BS0TSOx~&8_Eg4%Z6wWRBq7GMmoXroQjGrt9PKgH2gPILQM)z>8hg z#;tAjYGcb>cTMBs)?{ftAZKY&G-b~53hLr%n`Z0u)l6dt!R%3QMCxVx<|Zg8@jf!f zjYSNpNG9#)q;;KEMr)s|b=BmX64ylNZK>^sUn0}>a}ZYJV7{d#5`=kz2l9ElCgZ1Z N)$6|&Vt!P|;UB4uoPz)W From f5733a4842773dd3820ae01ef1fe861186ec3180 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Mon, 14 Mar 2016 22:02:34 +0530 Subject: [PATCH 04/62] correction in files --- MANIFEST.in | 1 + config.py | 54 +++++++++++++++++++ controllers/root.py | 22 -------- __init__.py => dockyard/__init__.py | 0 app.py => dockyard/app.py | 0 .../controllers}/__init__.py | 0 dockyard/controllers/root.py | 8 +++ {model => dockyard/model}/__init__.py | 0 {tests => dockyard/tests}/__init__.py | 0 {tests => dockyard/tests}/config.py | 0 {tests => dockyard/tests}/test_functional.py | 0 {tests => dockyard/tests}/test_units.py | 0 setup.cfg | 6 +++ setup.py | 22 ++++++++ 14 files changed, 91 insertions(+), 22 deletions(-) create mode 100644 MANIFEST.in create mode 100644 config.py delete mode 100644 controllers/root.py rename __init__.py => dockyard/__init__.py (100%) rename app.py => dockyard/app.py (100%) rename {controllers => dockyard/controllers}/__init__.py (100%) create mode 100644 dockyard/controllers/root.py rename {model => dockyard/model}/__init__.py (100%) rename {tests => dockyard/tests}/__init__.py (100%) rename {tests => dockyard/tests}/config.py (100%) rename {tests => dockyard/tests}/test_functional.py (100%) rename {tests => dockyard/tests}/test_units.py (100%) create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..c922f11 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include public * diff --git a/config.py b/config.py new file mode 100644 index 0000000..81fedcf --- /dev/null +++ b/config.py @@ -0,0 +1,54 @@ +# Server Specific Configurations +server = { + 'port': '8080', + 'host': '0.0.0.0' +} + +# Pecan Application Configurations +app = { + 'root': 'dockyard.controllers.root.RootController', + 'modules': ['dockyard'], + 'static_root': '%(confdir)s/public', + 'template_path': '%(confdir)s/dockyard/templates', + 'debug': True, + 'errors': { + 404: '/error/404', + '__force_dict__': True + } +} + +logging = { + 'root': {'level': 'INFO', 'handlers': ['console']}, + 'loggers': { + 'dockyard': {'level': 'DEBUG', 'handlers': ['console'], 'propagate': False}, + 'pecan': {'level': 'DEBUG', 'handlers': ['console'], 'propagate': False}, + 'py.warnings': {'handlers': ['console']}, + '__force_dict__': True + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'color' + } + }, + 'formatters': { + 'simple': { + 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' + '[%(threadName)s] %(message)s') + }, + 'color': { + '()': 'pecan.log.ColorFormatter', + 'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]' + '[%(threadName)s] %(message)s'), + '__force_dict__': True + } + } +} + +# Custom Configurations must be in Python dictionary format:: +# +# foo = {'bar':'baz'} +# +# All configurations are accessible at:: +# pecan.conf diff --git a/controllers/root.py b/controllers/root.py deleted file mode 100644 index bc1e72b..0000000 --- a/controllers/root.py +++ /dev/null @@ -1,22 +0,0 @@ -from pecan import expose, redirect -from webob.exc import status_map - - -class RootController(object): - - @expose(generic=True, template='index.html') - def index(self): - return dict() - - @index.when(method='POST') - def index_post(self, q): - redirect('http://pecan.readthedocs.org/en/latest/search.html?q=%s' % q) - - @expose('error.html') - def error(self, status): - try: - status = int(status) - except ValueError: # pragma: no cover - status = 500 - message = getattr(status_map.get(status), 'explanation', '') - return dict(status=status, message=message) diff --git a/__init__.py b/dockyard/__init__.py similarity index 100% rename from __init__.py rename to dockyard/__init__.py diff --git a/app.py b/dockyard/app.py similarity index 100% rename from app.py rename to dockyard/app.py diff --git a/controllers/__init__.py b/dockyard/controllers/__init__.py similarity index 100% rename from controllers/__init__.py rename to dockyard/controllers/__init__.py diff --git a/dockyard/controllers/root.py b/dockyard/controllers/root.py new file mode 100644 index 0000000..07c78c9 --- /dev/null +++ b/dockyard/controllers/root.py @@ -0,0 +1,8 @@ +from pecan import expose + + +class RootController(object): + + @expose() + def index(self): + return dict() diff --git a/model/__init__.py b/dockyard/model/__init__.py similarity index 100% rename from model/__init__.py rename to dockyard/model/__init__.py diff --git a/tests/__init__.py b/dockyard/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to dockyard/tests/__init__.py diff --git a/tests/config.py b/dockyard/tests/config.py similarity index 100% rename from tests/config.py rename to dockyard/tests/config.py diff --git a/tests/test_functional.py b/dockyard/tests/test_functional.py similarity index 100% rename from tests/test_functional.py rename to dockyard/tests/test_functional.py diff --git a/tests/test_units.py b/dockyard/tests/test_units.py similarity index 100% rename from tests/test_units.py rename to dockyard/tests/test_units.py diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..aa7fc1f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[nosetests] +match=^test +where=dockyard +nocapture=1 +cover-package=dockyard +cover-erase=1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e8a5ab0 --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +try: + from setuptools import setup, find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + from setuptools import setup, find_packages + +setup( + name='dockyard', + version='0.1', + description='', + author='', + author_email='', + install_requires=[ + "pecan", + ], + test_suite='dockyard', + zip_safe=False, + include_package_data=True, + packages=find_packages(exclude=['ez_setup']) +) From 0d3cf5be9f98f332738ff09f0d022e6e40e414cb Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Tue, 15 Mar 2016 19:02:17 +0530 Subject: [PATCH 05/62] Added basic structure --- dockyard/controllers/root.py | 20 ++++++++++++++++++++ dockyard/controllers/v1/__init__.py | 14 ++++++++++++++ dockyard/controllers/v1/container.py | 3 +++ dockyard/controllers/v1/image.py | 3 +++ dockyard/controllers/v1/information.py | 10 ++++++++++ dockyard/controllers/v1/network.py | 3 +++ dockyard/controllers/v1/volume.py | 3 +++ 7 files changed, 56 insertions(+) create mode 100644 dockyard/controllers/v1/__init__.py create mode 100644 dockyard/controllers/v1/container.py create mode 100644 dockyard/controllers/v1/image.py create mode 100644 dockyard/controllers/v1/information.py create mode 100644 dockyard/controllers/v1/network.py create mode 100644 dockyard/controllers/v1/volume.py diff --git a/dockyard/controllers/root.py b/dockyard/controllers/root.py index 07c78c9..32ccae7 100644 --- a/dockyard/controllers/root.py +++ b/dockyard/controllers/root.py @@ -1,8 +1,28 @@ from pecan import expose +from pecan import rest class RootController(object): + self._version = ['v1.0'] + # List of the allowed versions of the API. + + self._default_version = 'v1.0' + # Default value of the API version. + @expose() def index(self): return dict() + + @expose() + def _route(self, args): + """Override default routing. + + It redirect to the default value of the dockyard API, if version + of the API is not specified or wrongly specified + """ + + if args[0] and args[0] not in self._version: + args = self._default_version + args + + return super(rest.RootController(), self)._route(args) diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py new file mode 100644 index 0000000..09dc35f --- /dev/null +++ b/dockyard/controllers/v1/__init__.py @@ -0,0 +1,14 @@ +from dockyard.controllers.v1 import information +from dockyard.controllers.v1 import version +from dockyard.controllers.v1 import container +from dockyard.controllers.v1 import image +from dockyard.controllers.v1 import network +from dockyard.controllers.v1 import volume + +class Controller(object): + info = information.Information() + version = information.Version() + containers = container.ContainerController() + images = image.ImageController() + networks = network.NetworkController() + volumes = volume.Volumes() diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py new file mode 100644 index 0000000..7146310 --- /dev/null +++ b/dockyard/controllers/v1/container.py @@ -0,0 +1,3 @@ +class ContainerController(object): + def __init__(self): + pass diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py new file mode 100644 index 0000000..4acdcc3 --- /dev/null +++ b/dockyard/controllers/v1/image.py @@ -0,0 +1,3 @@ +class ImageController(object): + def __init__(self): + pass diff --git a/dockyard/controllers/v1/information.py b/dockyard/controllers/v1/information.py new file mode 100644 index 0000000..f55c3ed --- /dev/null +++ b/dockyard/controllers/v1/information.py @@ -0,0 +1,10 @@ +from pecan import expose + +class Information(object): + def __init__(self): + pass + +class Version(object): + @expose() + def version(self): + pass diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py new file mode 100644 index 0000000..3e243ea --- /dev/null +++ b/dockyard/controllers/v1/network.py @@ -0,0 +1,3 @@ +class NetworkController(object): + def __init__(self): + pass diff --git a/dockyard/controllers/v1/volume.py b/dockyard/controllers/v1/volume.py new file mode 100644 index 0000000..f26eb11 --- /dev/null +++ b/dockyard/controllers/v1/volume.py @@ -0,0 +1,3 @@ +class VolumeController(object): + def __init__(self): + pass From 76e47a7f2fda3935cdf1e21bf85a6bfd00105be1 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Wed, 16 Mar 2016 21:58:49 +0530 Subject: [PATCH 06/62] Changed code structure --- config.py | 54 -------------------------- dockyard/api/__init__.py | 0 dockyard/api/app.py | 42 ++++++++++++++++++++ dockyard/api/config.py | 24 ++++++++++++ dockyard/app.py | 14 ------- dockyard/cmd/__init__.py | 0 dockyard/cmd/api.py | 15 +++++++ dockyard/controllers/root.py | 17 ++++---- dockyard/controllers/v1/__init__.py | 3 +- dockyard/controllers/v1/information.py | 8 +++- 10 files changed, 97 insertions(+), 80 deletions(-) delete mode 100644 config.py create mode 100644 dockyard/api/__init__.py create mode 100644 dockyard/api/app.py create mode 100644 dockyard/api/config.py delete mode 100644 dockyard/app.py create mode 100644 dockyard/cmd/__init__.py create mode 100644 dockyard/cmd/api.py diff --git a/config.py b/config.py deleted file mode 100644 index 81fedcf..0000000 --- a/config.py +++ /dev/null @@ -1,54 +0,0 @@ -# Server Specific Configurations -server = { - 'port': '8080', - 'host': '0.0.0.0' -} - -# Pecan Application Configurations -app = { - 'root': 'dockyard.controllers.root.RootController', - 'modules': ['dockyard'], - 'static_root': '%(confdir)s/public', - 'template_path': '%(confdir)s/dockyard/templates', - 'debug': True, - 'errors': { - 404: '/error/404', - '__force_dict__': True - } -} - -logging = { - 'root': {'level': 'INFO', 'handlers': ['console']}, - 'loggers': { - 'dockyard': {'level': 'DEBUG', 'handlers': ['console'], 'propagate': False}, - 'pecan': {'level': 'DEBUG', 'handlers': ['console'], 'propagate': False}, - 'py.warnings': {'handlers': ['console']}, - '__force_dict__': True - }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'color' - } - }, - 'formatters': { - 'simple': { - 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' - '[%(threadName)s] %(message)s') - }, - 'color': { - '()': 'pecan.log.ColorFormatter', - 'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]' - '[%(threadName)s] %(message)s'), - '__force_dict__': True - } - } -} - -# Custom Configurations must be in Python dictionary format:: -# -# foo = {'bar':'baz'} -# -# All configurations are accessible at:: -# pecan.conf diff --git a/dockyard/api/__init__.py b/dockyard/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/api/app.py b/dockyard/api/app.py new file mode 100644 index 0000000..bb42e98 --- /dev/null +++ b/dockyard/api/app.py @@ -0,0 +1,42 @@ +import pecan + +from dockyard.api import config as api_config +from oslo_config import cfg +from pecan import make_app + +from dockyard import model + + +API_SERVICE_OPT = [ + cfg.PortOpt('port', + default=5869, + help='Port for the dockyard service.'), + cfg.IPOpt('host', + default='127.0.0.1', + help='Listening address for dockyard service'), +] + +CONF = cfg.CONF +opt_group = cfg.OptGroup(name='default', + title='Group for the default values for dockyard api') +CONF.register_group(opt_group) +CONF.register_opts(API_SERVICE_OPT, opt_group) + +def get_pecan_config(): + # Set up the pecan configuration + filename = api_config.__file__.replace('.pyc', '.py') + return pecan.configuration.conf_from_file(filename) + +def setup_app(config=None): + + if not config: + config = get_pecan_config() + + model.init_model() + app_conf = dict(config.app) + + return make_app( + app_conf.pop('root'), + logging=getattr(config, 'logging', {}), + **app_conf + ) diff --git a/dockyard/api/config.py b/dockyard/api/config.py new file mode 100644 index 0000000..7fa54d4 --- /dev/null +++ b/dockyard/api/config.py @@ -0,0 +1,24 @@ +# Pecan Application Configurations +app = { + 'root': 'dockyard.controllers.root.RootController', + 'modules': ['dockyard', 'dockyard.api'], + 'static_root': '%(confdir)s/public', + 'template_path': '%(confdir)s/dockyard/templates', + 'debug': True, + 'errors': { + 404: '/error/404', + '__force_dict__': True + }, + 'acl_public_routes': [ + '/', + '/v1', + ], + +} + +# Custom Configurations must be in Python dictionary format:: +# +# foo = {'bar':'baz'} +# +# All configurations are accessible at:: +# pecan.conf diff --git a/dockyard/app.py b/dockyard/app.py deleted file mode 100644 index 23225ba..0000000 --- a/dockyard/app.py +++ /dev/null @@ -1,14 +0,0 @@ -from pecan import make_app -from dockyard import model - - -def setup_app(config): - - model.init_model() - app_conf = dict(config.app) - - return make_app( - app_conf.pop('root'), - logging=getattr(config, 'logging', {}), - **app_conf - ) diff --git a/dockyard/cmd/__init__.py b/dockyard/cmd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/cmd/api.py b/dockyard/cmd/api.py new file mode 100644 index 0000000..5d667b4 --- /dev/null +++ b/dockyard/cmd/api.py @@ -0,0 +1,15 @@ +from wsgiref import simple_server + +from oslo_config import cfg +from oslo_log import log as logging +from dockyard.api import app as api_app + +def main(): + app = api_app.setup_app() + + # create the wsgi server and start it + host, port = cfg.CONF.default.host, cfg.CONF.default.port + srv = simple_server.make_server(host, port, app) + srv.serve_forever() + +main() diff --git a/dockyard/controllers/root.py b/dockyard/controllers/root.py index 32ccae7..8e96847 100644 --- a/dockyard/controllers/root.py +++ b/dockyard/controllers/root.py @@ -1,18 +1,19 @@ -from pecan import expose -from pecan import rest +from pecan import expose, rest +from dockyard.controllers import v1 -class RootController(object): +class RootController(rest.RestController): - self._version = ['v1.0'] + _version = ['v1'] # List of the allowed versions of the API. - self._default_version = 'v1.0' + _default_version = 'v1' # Default value of the API version. + v1 = v1.Controller() @expose() def index(self): - return dict() + return "OK\n" @expose() def _route(self, args): @@ -23,6 +24,6 @@ def _route(self, args): """ if args[0] and args[0] not in self._version: - args = self._default_version + args + args = [self._default_version] + args - return super(rest.RootController(), self)._route(args) + return super(RootController, self)._route(args) diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py index 09dc35f..c9036ed 100644 --- a/dockyard/controllers/v1/__init__.py +++ b/dockyard/controllers/v1/__init__.py @@ -1,5 +1,4 @@ from dockyard.controllers.v1 import information -from dockyard.controllers.v1 import version from dockyard.controllers.v1 import container from dockyard.controllers.v1 import image from dockyard.controllers.v1 import network @@ -11,4 +10,4 @@ class Controller(object): containers = container.ContainerController() images = image.ImageController() networks = network.NetworkController() - volumes = volume.Volumes() + volumes = volume.VolumeController() diff --git a/dockyard/controllers/v1/information.py b/dockyard/controllers/v1/information.py index f55c3ed..9d55764 100644 --- a/dockyard/controllers/v1/information.py +++ b/dockyard/controllers/v1/information.py @@ -3,8 +3,12 @@ class Information(object): def __init__(self): pass + + @expose() + def index(self): + return "Info" class Version(object): @expose() - def version(self): - pass + def index(self): + return "version" From 33b69b626a585114e9eb9a49491f6112a05c6731 Mon Sep 17 00:00:00 2001 From: karelyatin Date: Thu, 17 Mar 2016 23:45:31 +0530 Subject: [PATCH 07/62] Added base module for curl request --- dockyard/common/__init__.py | 0 dockyard/common/base.py | 245 +++++++++++++++++++++++++ dockyard/controllers/v1/__init__.py | 6 +- dockyard/controllers/v1/information.py | 22 ++- 4 files changed, 262 insertions(+), 11 deletions(-) create mode 100644 dockyard/common/__init__.py create mode 100644 dockyard/common/base.py diff --git a/dockyard/common/__init__.py b/dockyard/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/common/base.py b/dockyard/common/base.py new file mode 100644 index 0000000..98b1dc1 --- /dev/null +++ b/dockyard/common/base.py @@ -0,0 +1,245 @@ +import sys +import io +import json + +from oslo_log import log as logging + +# python 2 and python 3 compatibility library +from six import iteritems + +try: + import urllib3 +except ImportError: + raise ImportError('python client requires urllib3') + +try: + # for python3 + from urllib.parse import urlencode +except ImportError: + # for python2 + from urllib import urlencode + + +logger = logging.getLogger(__name__) + + +class RESTResponse(io.IOBase): + + def __init__(self, resp): + self.urllib3_response = resp + self.status = resp.status + self.reason = resp.reason + self.data = resp.data + + def getheaders(self): + """ + Returns a dictionary of the response headers. + """ + return self.urllib3_response.getheaders() + + def getheader(self, name, default=None): + """ + Returns a given response header. + """ + return self.urllib3_response.getheader(name, default) + + +class RESTClientObject(object): + + def __init__(self, pools_size=4): + # http pool manager + self.pool_manager = urllib3.PoolManager( + num_pools=pools_size + ) + + def agent(self, url): + """ + Use `urllib3.util.parse_url` for backward compatibility. + Return proper pool manager for the http/https schemes. + """ + url = urllib3.util.parse_url(url) + scheme = url.scheme + if scheme == 'https': + return self.ssl_pool_manager + else: + return self.pool_manager + + def request(self, method, url, query_params=None, headers=None, + body=None, post_params=None): + """ + :param method: http request method + :param url: http request url + :param query_params: query parameters in the url + :param headers: http request headers + :param body: request json body, for `application/json` + :param post_params: request post parameters, + `application/x-www-form-urlencode` + and `multipart/form-data` + """ + method = method.upper() + assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH'] + + if post_params and body: + raise ValueError( + "body parameter cannot be used with post_params parameter." + ) + + post_params = post_params or {} + headers = headers or {} + + if 'Content-Type' not in headers: + headers['Content-Type'] = 'application/json' + + # For `POST`, `PUT`, `PATCH` + if method in ['POST', 'PUT', 'PATCH']: + if query_params: + url += '?' + urlencode(query_params) + if headers['Content-Type'] == 'application/json': + response = self.agent(url).request(method, url, + body=json.dumps(body), + headers=headers) + if headers['Content-Type'] == 'application/x-www-form-urlencoded': + response = self.agent(url).request(method, url, + encode_multipart=False, + headers=headers) + if headers['Content-Type'] == 'multipart/form-data': + # must del headers['Content-Type'], or the correct Content-Type + # which generated by urllib3 will be overwritten. + del headers['Content-Type'] + response = self.agent(url).request(method, url, + encode_multipart=True, + headers=headers) + # For `GET`, `HEAD`, `DELETE` + else: + response = self.agent(url).request(method, url, + headers=headers) + response = RESTResponse(response) + + # In the python 3, the response.data is bytes. + # we need to decode it to string. + if sys.version_info > (3,): + response.data = response.data.decode('utf8') + + # log response body + logger.debug("response body: %s" % response.data) + + if response.status not in range(200, 206): + raise ApiException(http_resp=response) + + return response + + def GET(self, url, headers=None, query_params=None): + return self.request("GET", url, + headers=headers, + query_params=query_params) + + def HEAD(self, url, headers=None, query_params=None): + return self.request("HEAD", url, + headers=headers, + query_params=query_params) + + def DELETE(self, url, headers=None, query_params=None): + return self.request("DELETE", url, + headers=headers, + query_params=query_params) + + def POST(self, url, headers=None, query_params=None, post_params=None, body=None): + return self.request("POST", url, + headers=headers, + query_params=query_params, + post_params=post_params, + body=body) + + def PUT(self, url, headers=None, query_params=None, post_params=None, body=None): + return self.request("PUT", url, + headers=headers, + query_params=query_params, + post_params=post_params, + body=body) + + def PATCH(self, url, headers=None, query_params=None, post_params=None, body=None): + return self.request("PATCH", url, + headers=headers, + query_params=query_params, + post_params=post_params, + body=body) + + +class ApiException(Exception): + + def __init__(self, status=None, reason=None, http_resp=None): + if http_resp: + self.status = http_resp.status + self.reason = http_resp.reason + self.body = http_resp.data + self.headers = http_resp.getheaders() + else: + self.status = status + self.reason = reason + self.body = None + self.headers = None + + def __str__(self): + """ + Custom error messages for exception + """ + error_message = "({0})\n"\ + "Reason: {1}\n".format(self.status, self.reason) + if self.headers: + error_message += "HTTP response headers: {0}\n".format(self.headers) + + if self.body: + error_message += "HTTP response body: {0}\n".format(self.body) + + return error_message + +class RESTClient(object): + """ + A class with methods to perform JSON requests. + """ + + def __init__(self): + self.IMPL = RESTClientObject() + + def request(self, *n, **kw): + """ + Perform a REST request and parse the response. + """ + return self.IMPL.request(*n, **kw) + + def GET(self, *n, **kw): + """ + Perform a GET request using `RESTClient.request()`. + """ + return self.IMPL.GET(*n, **kw) + + def HEAD(self, *n, **kw): + """ + Perform a HEAD request using `RESTClient.request()`. + """ + return self.IMPL.GET(*n, **kw) + + def POST(self, *n, **kw): + """ + Perform a POST request using `RESTClient.request()` + """ + return self.IMPL.POST(*n, **kw) + + def PUT(self, *n, **kw): + """ + Perform a PUT request using `RESTClient.request()` + """ + return self.IMPL.PUT(*n, **kw) + + def PATCH(self, *n, **kw): + """ + Perform a PATCH request using `RESTClient.request()` + """ + return self.IMPL.PATCH(*n, **kw) + + def DELETE(self, *n, **kw): + """ + Perform a DELETE request using `RESTClient.request()` + """ + return self.IMPL.DELETE(*n, **kw) + diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py index c9036ed..ac26c61 100644 --- a/dockyard/controllers/v1/__init__.py +++ b/dockyard/controllers/v1/__init__.py @@ -3,8 +3,10 @@ from dockyard.controllers.v1 import image from dockyard.controllers.v1 import network from dockyard.controllers.v1 import volume - -class Controller(object): + +from pecan import rest + +class Controller(rest.RestController): info = information.Information() version = information.Version() containers = container.ContainerController() diff --git a/dockyard/controllers/v1/information.py b/dockyard/controllers/v1/information.py index 9d55764..b0a8ba3 100644 --- a/dockyard/controllers/v1/information.py +++ b/dockyard/controllers/v1/information.py @@ -1,14 +1,18 @@ from pecan import expose +from pecan import rest -class Information(object): - def __init__(self): - pass - +import json + +from dockyard.common import base + +class Information(rest.RestController): @expose() - def index(self): - return "Info" + def get(self): + rest_client = base.RESTClient() + return rest_client.GET('http://10.0.0.12:3333/info').data -class Version(object): +class Version(rest.RestController): @expose() - def index(self): - return "version" + def get(self): + rest_client = base.RESTClient() + return rest_client.GET('http://10.0.0.12:3333/version').data From 6103cd308b3e5d7df8702e40e3cfb3e6ed88e775 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Thu, 17 Mar 2016 23:50:43 +0530 Subject: [PATCH 08/62] Change in structure --- dockyard/common/container/Container.py | 0 dockyard/common/container/__init__.py | 0 dockyard/common/image/__init__.py | 0 dockyard/common/network/__init__.py | 0 dockyard/common/utils.py | 0 dockyard/common/volume/__init__.py | 0 dockyard/controllers/v1/container.py | 4 +++- dockyard/controllers/v1/image.py | 27 ++++++++++++++++++++++++++ dockyard/controllers/v1/network.py | 22 +++++++++++++++++++++ dockyard/controllers/v1/volume.py | 16 +++++++++++++++ 10 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 dockyard/common/container/Container.py create mode 100644 dockyard/common/container/__init__.py create mode 100644 dockyard/common/image/__init__.py create mode 100644 dockyard/common/network/__init__.py create mode 100644 dockyard/common/utils.py create mode 100644 dockyard/common/volume/__init__.py diff --git a/dockyard/common/container/Container.py b/dockyard/common/container/Container.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/common/container/__init__.py b/dockyard/common/container/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/common/image/__init__.py b/dockyard/common/image/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/common/network/__init__.py b/dockyard/common/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/common/volume/__init__.py b/dockyard/common/volume/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py index 7146310..72fa7f3 100644 --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -1,3 +1,5 @@ -class ContainerController(object): +from pecan import rest + +class ContainerController(rest.Controller): def __init__(self): pass diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py index 4acdcc3..85d53fd 100644 --- a/dockyard/controllers/v1/image.py +++ b/dockyard/controllers/v1/image.py @@ -1,3 +1,30 @@ +from pecan import expose, route + +class Image(object): + def __init__(self): + pass + + @expose() + def push(self, _id): + return "PUSH %d Image" + + @expose(generic=True) + def create(self, *args): + return "Invalid Create" + + @create.when(method="POST") + def create_POST(self, *args): + return "Valid create" + + @expose() + def json(self, _id=None): + return "details" + class ImageController(object): def __init__(self): pass + + @expose() + def _lookup(self, _id, op): + return Image(), (op + _id); +route('json', Image().json) diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py index 3e243ea..3966005 100644 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -1,3 +1,25 @@ +from pecan import expose + +class Network(object): + def __init__(self): + return + class NetworkController(object): def __init__(self): pass + + @expose(generic=True) + def index(self, *name_or_id, **kwargs): + return "Get name of the networks:" + + @index.when(method='POST') + def set_network(self, _id, *connect): + return "Set network" + + @expose(generic=True) + def create(self): + return "Invalid" + + @create.when(method="POST") + def create(self, *args): + return "create" diff --git a/dockyard/controllers/v1/volume.py b/dockyard/controllers/v1/volume.py index f26eb11..c019989 100644 --- a/dockyard/controllers/v1/volume.py +++ b/dockyard/controllers/v1/volume.py @@ -1,3 +1,19 @@ class VolumeController(object): def __init__(self): pass + + @expose(generic=True) + def index(self, name=None): + return "index" + + @index.when(method='DELETE') + def index_delete(self, name) + return "Delete Volume" + + @expose(generic=True) + def create(self): + return "create" + + @create.when(method='POST') + def create(self): + return "create" From 0d56ca3074b57f9259ae9541005ff7472ec81790 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Mon, 21 Mar 2016 21:22:08 +0530 Subject: [PATCH 09/62] Added structure for Scheduler --- .../Container.py => cluster/__init__.py} | 0 dockyard/common/cluster/cluster.py | 14 +++ .../common/cluster/membership/__init__.py | 0 dockyard/common/container/container.py | 35 +++++++ .../common/container/scheduler/__init__.py | 0 dockyard/common/container/scheduler/base.py | 15 +++ .../common/container/scheduler/round_robin.py | 16 ++++ dockyard/common/image/image.py | 17 ++++ dockyard/common/link.py | 12 +++ dockyard/common/network/network.py | 14 +++ dockyard/common/utils.py | 11 +++ dockyard/common/volume/volume.py | 14 +++ dockyard/controllers/v1/__init__.py | 13 +-- dockyard/controllers/v1/cluster.py | 47 +++++++++ dockyard/controllers/v1/container.py | 96 ++++++++++++++++++- dockyard/controllers/v1/image.py | 26 +++-- dockyard/controllers/v1/information.py | 16 ++-- dockyard/controllers/v1/network.py | 41 +++++--- dockyard/controllers/v1/volume.py | 39 ++++++-- 19 files changed, 384 insertions(+), 42 deletions(-) rename dockyard/common/{container/Container.py => cluster/__init__.py} (100%) create mode 100644 dockyard/common/cluster/cluster.py create mode 100644 dockyard/common/cluster/membership/__init__.py create mode 100644 dockyard/common/container/container.py create mode 100644 dockyard/common/container/scheduler/__init__.py create mode 100644 dockyard/common/container/scheduler/base.py create mode 100644 dockyard/common/container/scheduler/round_robin.py create mode 100644 dockyard/common/image/image.py create mode 100644 dockyard/common/link.py create mode 100644 dockyard/common/network/network.py create mode 100644 dockyard/common/volume/volume.py create mode 100644 dockyard/controllers/v1/cluster.py diff --git a/dockyard/common/container/Container.py b/dockyard/common/cluster/__init__.py similarity index 100% rename from dockyard/common/container/Container.py rename to dockyard/common/cluster/__init__.py diff --git a/dockyard/common/cluster/cluster.py b/dockyard/common/cluster/cluster.py new file mode 100644 index 0000000..612cea6 --- /dev/null +++ b/dockyard/common/cluster/cluster.py @@ -0,0 +1,14 @@ +from oslo_config import cfg + +class Cluster(object): + def __init__(self): + pass + + def register(self, cluster_id, host_ip, port): + pass + + def unregister(self, cluster_id, host_ip): + pass + + def get_hosts(self): + return cfg.CONF.membership.hosts diff --git a/dockyard/common/cluster/membership/__init__.py b/dockyard/common/cluster/membership/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py new file mode 100644 index 0000000..b9de3fe --- /dev/null +++ b/dockyard/common/container/container.py @@ -0,0 +1,35 @@ +from dockyard.common import base + +class Container(object): + def __init__(self): + self.rest_client = base.RESTClient() + + def list(self, name_id=None): + return self.rest_client.GET("http://127.0.0.1:3333/containers/json").data + + def stats(self, _id): + return "stats" + + def archive(self, _id): + return "archive" + + def upload(self, _id): + return "upload" + + def copy(self, _id): + return "copy" + + def logs(self, _id): + return "logs" + + def start(self, _id): + return "start" + + def exe(self, _id): + return "exec" + + def attach(self, _id): + return "attach" + + def rename(self, _id): + return "rename" diff --git a/dockyard/common/container/scheduler/__init__.py b/dockyard/common/container/scheduler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/common/container/scheduler/base.py b/dockyard/common/container/scheduler/base.py new file mode 100644 index 0000000..7650b45 --- /dev/null +++ b/dockyard/common/container/scheduler/base.py @@ -0,0 +1,15 @@ +import abc + + +class Scheduler(object): + @abc.abstractmethod + def get_host(self, *args, **kwargs): + """This method returns host information to launch + container. + + It expects specification of docker container and + interact with membership managment protocol to + find the host to run a container. + + Every scheduler should have this method. + """ diff --git a/dockyard/common/container/scheduler/round_robin.py b/dockyard/common/container/scheduler/round_robin.py new file mode 100644 index 0000000..b932055 --- /dev/null +++ b/dockyard/common/container/scheduler/round_robin.py @@ -0,0 +1,16 @@ +from oslo_config import cfg + +from dockyard.common.cluster import cluster +from dockyard.common.container.scheduler.base import Scheduler + + +class RoundRobinScheduler(Scheduler): + count = -1 + def __init__(self): + self.cluster = cluster.Cluster() + + def get_host(self): + hosts = self.cluster.get_hosts() + num_hosts = len(hosts) + self.count += 1 + return (hosts[self.count % num_hosts]) diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py new file mode 100644 index 0000000..025f5ad --- /dev/null +++ b/dockyard/common/image/image.py @@ -0,0 +1,17 @@ +from dockyard.common import base, link +from dockyard.common import utils + +class Image(object): + def __init__(self): + self.rest_client = base.RESTClient() + + def list(self, _id=None): + host = utils.get_host() + ln = link.make_url(host=host, protocol='http', url='/images/json') + return self.rest_client.GET(ln).data + + def create(self, _id): + return "Valid create\n" + + def push(self, _id): + return (("PUSH %s Image\n") % (_id)) diff --git a/dockyard/common/link.py b/dockyard/common/link.py new file mode 100644 index 0000000..2c5de06 --- /dev/null +++ b/dockyard/common/link.py @@ -0,0 +1,12 @@ +def make_url(host='127.0.0.1', port=None, protocol='http', url=None): + head = protocol + '://' + head = head + host + if port: + head = head + ":" + port + + if url: + url = head + url + else: + url = head + + return url diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py new file mode 100644 index 0000000..a362825 --- /dev/null +++ b/dockyard/common/network/network.py @@ -0,0 +1,14 @@ +from dockyard.common import base + +class Network(object): + def __init__(self): + self.rest_client = base.RESTClient() + + def list(self, name_or_id): + return str(self.rest_client.GET("http://127.0.0.1:3333/networks/").data) + + def set_network(self, _id, *connect): + return "SET network" + + def create(self, *args): + return "Create Network" diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index e69de29..4798a7a 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -0,0 +1,11 @@ +from oslo_config import cfg + +from dockyard.common.container.scheduler.round_robin import RoundRobinScheduler + +def get_config(group, option): + CONF = cfg.CONF + return CONF.group.option + + +def get_host(): + return RoundRobinScheduler().get_host() diff --git a/dockyard/common/volume/volume.py b/dockyard/common/volume/volume.py new file mode 100644 index 0000000..bb80cc9 --- /dev/null +++ b/dockyard/common/volume/volume.py @@ -0,0 +1,14 @@ +from dockyard.common import base + +class Volume(object): + def __init__(self): + self.rest_client = base.RESTClient() + + def list(self, name=None): + return self.rest_client.GET("http://localhost:3333/volumes").data + + def delete(self, name): + return "Delete Volume" + + def create(self): + return "POST create" diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py index ac26c61..ee56ed3 100644 --- a/dockyard/controllers/v1/__init__.py +++ b/dockyard/controllers/v1/__init__.py @@ -1,9 +1,9 @@ -from dockyard.controllers.v1 import information -from dockyard.controllers.v1 import container -from dockyard.controllers.v1 import image -from dockyard.controllers.v1 import network -from dockyard.controllers.v1 import volume - +from dockyard.controllers.v1 import information +from dockyard.controllers.v1 import container +from dockyard.controllers.v1 import image +from dockyard.controllers.v1 import network +from dockyard.controllers.v1 import volume +from dockyard.controllers.v1 import cluster from pecan import rest class Controller(rest.RestController): @@ -13,3 +13,4 @@ class Controller(rest.RestController): images = image.ImageController() networks = network.NetworkController() volumes = volume.VolumeController() + clusters = cluster.ClusterController() diff --git a/dockyard/controllers/v1/cluster.py b/dockyard/controllers/v1/cluster.py new file mode 100644 index 0000000..826e3b6 --- /dev/null +++ b/dockyard/controllers/v1/cluster.py @@ -0,0 +1,47 @@ +from pecan import expose, abort +from oslo_config import cfg + +from dockyard.common.cluster import cluster + +class Cluster(object): + def __init__(self): + self.cluster = cluster.Cluster() + + @expose(generic=True) + def register(self, *args): + abort(404) + + @register.when(method="PUT") + def register_POST(self, cluster_id, host_ip, port): + return self.cluster.register(cluster_id, host_ip, port) + + @expose(generic=True) + def unregister(self, *args): + abort(404) + + @unregister.when(method="DELETE") + def unregister_POST(self, cluster_id, host_ip): + return self.cluster.unregister(cluster_id, host_ip) + +class ClusterController(object): + def __init__(self): + HOST_SERVICE_OPT = [ + cfg.ListOpt('hosts', + default='127.0.0.1:3333', + help='Listening address of docker service'), + ] + + CONF = cfg.CONF + opt_group = cfg.OptGroup(name='membership', + title='Group for membership of docker services') + CONF.register_group(opt_group) + CONF.register_opts(HOST_SERVICE_OPT, opt_group) + + @expose() + def _lookup(self, op, cluster_id, host_ip, port=None): + new_url = [op, cluster_id, host_ip] + + if port: + new_url.append(port) + + return Cluster(), tuple(new_url) diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py index 72fa7f3..02b14d4 100644 --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -1,5 +1,97 @@ -from pecan import rest +from pecan import expose, abort -class ContainerController(rest.Controller): +from dockyard.common.container import container + +class Container(object): + def __init__(self): + self.container = container.Container() + + @expose(generic=True) + def index(self): + return "index" + + @expose() + def json(self, name_id=None): + return self.container.list(name_id) + + @expose() + def stats(self, _id): + return self.container.stats(_id) + + @expose(generic=True) + def archive(self, _id): + return self.container.archive(_id) + + @archive.when(method="PUT") + def upload(self, _id): + return self.container.upload(_id) + + @expose(generic=True) + def copy(self, _id): + abort(404) + + @copy.when(method="POST") + def copy_POST(self, _id): + return self.container.copy(_id) + + @expose(generic=True) + def logs(self, _id): + return self.container.logs(_id) + + @expose(generic=True) + def start(self): + abort(404) + + @start.when(method="POST") + def start_POST(self, _id): + return self.container.start(_id) + + @expose(generic=True, route='exec') + def exe(self): + abort(404) + + @exe.when(method="POST") + def exec_POST(self, _id): + return self.container.exe(_id) + + @expose(generic=True) + def attach(self): + abort(404) + + @attach.when(method="POST") + def attach_POST(self, _id): + return self.container.attach(_id) + + @expose(generic=True) + def rename(self): + abort(404) + + @rename.when(method="POST") + def rename_POST(self, _id): + return self.container.rename(_id) + + + +class ContainerController(object): def __init__(self): pass + + @expose(generic=True) + def create(self, *args): + abort(404) + + @create.when(method="POST") + def create_container(self, *args): + return self.container.create(args) + + @expose() + def json(self): + return Container().json() + + def _lookup(self, _id=None, op=None): + if op != None: + new_url = (op, _id) + else: + new_url = tuple([_id]) + + return Container(), new_url; diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py index 85d53fd..2b007d2 100644 --- a/dockyard/controllers/v1/image.py +++ b/dockyard/controllers/v1/image.py @@ -1,30 +1,36 @@ -from pecan import expose, route +from pecan import expose, route, abort + +from dockyard.common.image import image class Image(object): def __init__(self): - pass - + self.image = image.Image() + super(Image, self).__init__() + @expose() def push(self, _id): - return "PUSH %d Image" + return self.image.push(_id) @expose(generic=True) def create(self, *args): - return "Invalid Create" + abort(404) @create.when(method="POST") def create_POST(self, *args): - return "Valid create" + return self.image.create(_id) @expose() def json(self, _id=None): - return "details" + return self.image.list(_id) class ImageController(object): def __init__(self): pass @expose() - def _lookup(self, _id, op): - return Image(), (op + _id); -route('json', Image().json) + def _lookup(self, _id, op=None): + if op != None: + new_url = (op, _id) + else: + new_url = tuple([_id]) + return Image(), new_url; diff --git a/dockyard/controllers/v1/information.py b/dockyard/controllers/v1/information.py index b0a8ba3..0bfd0cc 100644 --- a/dockyard/controllers/v1/information.py +++ b/dockyard/controllers/v1/information.py @@ -1,18 +1,22 @@ from pecan import expose from pecan import rest -import json - from dockyard.common import base class Information(rest.RestController): + def __init__(self): + self.rest_client = base.RESTClient() + super(Information, self).__init__() + @expose() def get(self): - rest_client = base.RESTClient() - return rest_client.GET('http://10.0.0.12:3333/info').data + return self.rest_client.GET('http://127.0.0.1:3333/info').data class Version(rest.RestController): + def __init__(self): + self.rest_client = base.RESTClient() + super(Version, self).__init__() + @expose() def get(self): - rest_client = base.RESTClient() - return rest_client.GET('http://10.0.0.12:3333/version').data + return self.rest_client.GET('http://127.0.0.1:3333/version').data diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py index 3966005..71b4082 100644 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -1,25 +1,44 @@ -from pecan import expose +from pecan import expose, abort + +from dockyard.common.network import network class Network(object): def __init__(self): - return - -class NetworkController(object): - def __init__(self): - pass + self.network = network.Network() @expose(generic=True) - def index(self, *name_or_id, **kwargs): - return "Get name of the networks:" + def index(self, name_or_id=None): + return self.network.list(name_or_id) @index.when(method='POST') def set_network(self, _id, *connect): - return "Set network" + return self.network.set_network(_id, connect) @expose(generic=True) def create(self): - return "Invalid" + abort(404) @create.when(method="POST") def create(self, *args): - return "create" + return self.network.create(args) + +class NetworkController(object): + def __init__(self): + pass + + @expose() + def _lookup(self, id_name=None, op=None): + new_url = [] + + if op: + new_url.append([op]) + + if id_name: + new_url.append([id_name]) + + if new_url: + new_url = tuple(new_url) + else: + new_url = tuple(['']) + + return Network(), new_url diff --git a/dockyard/controllers/v1/volume.py b/dockyard/controllers/v1/volume.py index c019989..f450824 100644 --- a/dockyard/controllers/v1/volume.py +++ b/dockyard/controllers/v1/volume.py @@ -1,19 +1,44 @@ -class VolumeController(object): +from pecan import expose + +from dockyard.common.volume import volume + +class Volume(object): def __init__(self): - pass + self.volume = volume.Volume() @expose(generic=True) def index(self, name=None): - return "index" + return self.volume.list() @index.when(method='DELETE') - def index_delete(self, name) - return "Delete Volume" + def index_DELETE(self, name): + return self.volume.delete() @expose(generic=True) def create(self): - return "create" + abort(404) @create.when(method='POST') def create(self): - return "create" + return self.volume.create() + +class VolumeController(object): + def __init__(self): + pass + + @expose() + def _lookup(self, id_name=None, op=None): + new_url = [] + + if op: + new_url.append([op]) + + if id_name: + new_url.append([id_name]) + + if new_url: + new_url = tuple(new_url) + else: + new_url = tuple(['']) + + return Volume(), new_url From b57661560b4c6d6a7f35067692388538d255a88d Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Tue, 22 Mar 2016 21:03:21 +0530 Subject: [PATCH 10/62] Image create --- .../common/container/scheduler/round_robin.py | 4 ++-- dockyard/common/image/image.py | 10 ++++----- dockyard/common/information/__init__.py | 0 dockyard/common/information/information.py | 21 +++++++++++++++++++ dockyard/common/link.py | 8 +++++++ dockyard/common/utils.py | 16 +++++++++++++- dockyard/controllers/v1/image.py | 4 ++-- dockyard/controllers/v1/information.py | 11 +++++----- 8 files changed, 58 insertions(+), 16 deletions(-) create mode 100644 dockyard/common/information/__init__.py create mode 100644 dockyard/common/information/information.py diff --git a/dockyard/common/container/scheduler/round_robin.py b/dockyard/common/container/scheduler/round_robin.py index b932055..1d2a0b2 100644 --- a/dockyard/common/container/scheduler/round_robin.py +++ b/dockyard/common/container/scheduler/round_robin.py @@ -12,5 +12,5 @@ def __init__(self): def get_host(self): hosts = self.cluster.get_hosts() num_hosts = len(hosts) - self.count += 1 - return (hosts[self.count % num_hosts]) + self.count = (self.count + 1) % num_hosts + return (hosts[self.count]) diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py index 025f5ad..175614f 100644 --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -6,12 +6,12 @@ def __init__(self): self.rest_client = base.RESTClient() def list(self, _id=None): - host = utils.get_host() - ln = link.make_url(host=host, protocol='http', url='/images/json') - return self.rest_client.GET(ln).data + return utils.dispatch_request(protocol='http', url='/images/json').data - def create(self, _id): - return "Valid create\n" + def create(self, fromImage, tag): + query = link.make_query_url(fromImage=fromImage, tag=tag) + url = (('/images/create?%s') % (query)) + return utils.dispatch_post_request(protocol='http', url=url) def push(self, _id): return (("PUSH %s Image\n") % (_id)) diff --git a/dockyard/common/information/__init__.py b/dockyard/common/information/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/common/information/information.py b/dockyard/common/information/information.py new file mode 100644 index 0000000..42a06d6 --- /dev/null +++ b/dockyard/common/information/information.py @@ -0,0 +1,21 @@ +from dockyard.common import base, link +from dockyard.common import utils + +class Information(object): + def __init__(self): + self.rest_client = base.RESTClient() + + def info(self): + host = utils.get_host() + ln = link.make_url(host=host, protocol='http', url='/info') + return self.rest_client.GET(ln).data + +class Version(object): + def __init__(self): + self.rest_client = base.RESTClient() + + def version(self): + host = utils.get_host() + ln = link.make_url(host=host, protocol='http', url='/version') + return self.rest_client.GET(ln).data + diff --git a/dockyard/common/link.py b/dockyard/common/link.py index 2c5de06..afc19e2 100644 --- a/dockyard/common/link.py +++ b/dockyard/common/link.py @@ -1,3 +1,4 @@ + def make_url(host='127.0.0.1', port=None, protocol='http', url=None): head = protocol + '://' head = head + host @@ -10,3 +11,10 @@ def make_url(host='127.0.0.1', port=None, protocol='http', url=None): url = head return url + +def make_query_url(**kwargs): + url = '' + for key, value in kwargs.iteritems(): + url += key + '=' + value + '&' + url = url[:-1] + return url diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index 4798a7a..44b7bb6 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -1,11 +1,25 @@ from oslo_config import cfg +from dockyard.common import base, link from dockyard.common.container.scheduler.round_robin import RoundRobinScheduler +rest_client = base.RESTClient() + def get_config(group, option): CONF = cfg.CONF return CONF.group.option - def get_host(): return RoundRobinScheduler().get_host() + +def get_link(url, protocol): + host = get_host() + return link.make_url(host=host, protocol=protocol, url=url) + +def dispatch_request(url, protocol): + ln = get_link(url, protocol) + return rest_client.GET(ln) + +def dispatch_post_request(url, protocol): + ln = get_link(url, protocol) + return rest_client.POST(ln) diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py index 2b007d2..0b23f78 100644 --- a/dockyard/controllers/v1/image.py +++ b/dockyard/controllers/v1/image.py @@ -16,8 +16,8 @@ def create(self, *args): abort(404) @create.when(method="POST") - def create_POST(self, *args): - return self.image.create(_id) + def create_POST(self, fromImage, tag='latest'): + return self.image.create(fromImage, tag) @expose() def json(self, _id=None): diff --git a/dockyard/controllers/v1/information.py b/dockyard/controllers/v1/information.py index 0bfd0cc..30bccca 100644 --- a/dockyard/controllers/v1/information.py +++ b/dockyard/controllers/v1/information.py @@ -2,21 +2,20 @@ from pecan import rest from dockyard.common import base +from dockyard.common.information import information class Information(rest.RestController): def __init__(self): - self.rest_client = base.RESTClient() - super(Information, self).__init__() + self.information = information.Information() @expose() def get(self): - return self.rest_client.GET('http://127.0.0.1:3333/info').data + return self.information.info() class Version(rest.RestController): def __init__(self): - self.rest_client = base.RESTClient() - super(Version, self).__init__() + self.information = information.Version() @expose() def get(self): - return self.rest_client.GET('http://127.0.0.1:3333/version').data + return self.information.version() From 5c1b300c4bb42106c8dfb99113e9c3a1490ea049 Mon Sep 17 00:00:00 2001 From: karelyatin Date: Wed, 23 Mar 2016 07:12:44 +0530 Subject: [PATCH 11/62] updated image create --- dockyard/common/base.py | 3 +-- dockyard/common/container/container.py | 7 +++++-- dockyard/common/image/image.py | 6 +++--- dockyard/common/network/network.py | 7 +++++-- dockyard/common/utils.py | 4 ++-- dockyard/common/volume/volume.py | 7 +++++-- dockyard/controllers/v1/__init__.py | 1 + dockyard/controllers/v1/image.py | 2 +- 8 files changed, 23 insertions(+), 14 deletions(-) diff --git a/dockyard/common/base.py b/dockyard/common/base.py index 98b1dc1..1881e27 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -96,7 +96,7 @@ def request(self, method, url, query_params=None, headers=None, url += '?' + urlencode(query_params) if headers['Content-Type'] == 'application/json': response = self.agent(url).request(method, url, - body=json.dumps(body), +# body=json.dumps(body), headers=headers) if headers['Content-Type'] == 'application/x-www-form-urlencoded': response = self.agent(url).request(method, url, @@ -119,7 +119,6 @@ def request(self, method, url, query_params=None, headers=None, # we need to decode it to string. if sys.version_info > (3,): response.data = response.data.decode('utf8') - # log response body logger.debug("response body: %s" % response.data) diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index b9de3fe..c19f520 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -1,11 +1,14 @@ -from dockyard.common import base +from dockyard.common import base, link +from dockyard.common import utils class Container(object): def __init__(self): self.rest_client = base.RESTClient() def list(self, name_id=None): - return self.rest_client.GET("http://127.0.0.1:3333/containers/json").data + host = utils.get_host() + ln = link.make_url(host=host, protocol='http', url='/containers/json') + return self.rest_client.GET(ln).data def stats(self, _id): return "stats" diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py index 175614f..33e27d3 100644 --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -9,9 +9,9 @@ def list(self, _id=None): return utils.dispatch_request(protocol='http', url='/images/json').data def create(self, fromImage, tag): - query = link.make_query_url(fromImage=fromImage, tag=tag) - url = (('/images/create?%s') % (query)) - return utils.dispatch_post_request(protocol='http', url=url) + url = '/images/create' + query = { "fromImage" : fromImage, "tag" : tag } + return utils.dispatch_post_request(protocol='http', url=url, query_params=query) def push(self, _id): return (("PUSH %s Image\n") % (_id)) diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py index a362825..b28a32f 100644 --- a/dockyard/common/network/network.py +++ b/dockyard/common/network/network.py @@ -1,11 +1,14 @@ -from dockyard.common import base +from dockyard.common import base, link +from dockyard.common import utils class Network(object): def __init__(self): self.rest_client = base.RESTClient() def list(self, name_or_id): - return str(self.rest_client.GET("http://127.0.0.1:3333/networks/").data) + host = utils.get_host() + ln = link.make_url(host=host, protocol='http', url='/networks/') + return str(self.rest_client.GET(ln).data) def set_network(self, _id, *connect): return "SET network" diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index 44b7bb6..5ea8b77 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -20,6 +20,6 @@ def dispatch_request(url, protocol): ln = get_link(url, protocol) return rest_client.GET(ln) -def dispatch_post_request(url, protocol): +def dispatch_post_request(url, protocol, query_params): ln = get_link(url, protocol) - return rest_client.POST(ln) + return rest_client.POST(url=ln, query_params=query_params) diff --git a/dockyard/common/volume/volume.py b/dockyard/common/volume/volume.py index bb80cc9..edce2e6 100644 --- a/dockyard/common/volume/volume.py +++ b/dockyard/common/volume/volume.py @@ -1,11 +1,14 @@ -from dockyard.common import base +from dockyard.common import base, link +from dockyard.common import utils class Volume(object): def __init__(self): self.rest_client = base.RESTClient() def list(self, name=None): - return self.rest_client.GET("http://localhost:3333/volumes").data + host = utils.get_host() + ln = link.make_url(host=host, protocol='http', url='/volumes') + return self.rest_client.GET(ln).data def delete(self, name): return "Delete Volume" diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py index ee56ed3..b92870a 100644 --- a/dockyard/controllers/v1/__init__.py +++ b/dockyard/controllers/v1/__init__.py @@ -5,6 +5,7 @@ from dockyard.controllers.v1 import volume from dockyard.controllers.v1 import cluster from pecan import rest +from pecan import expose class Controller(rest.RestController): info = information.Information() diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py index 0b23f78..b2b20e3 100644 --- a/dockyard/controllers/v1/image.py +++ b/dockyard/controllers/v1/image.py @@ -17,7 +17,7 @@ def create(self, *args): @create.when(method="POST") def create_POST(self, fromImage, tag='latest'): - return self.image.create(fromImage, tag) + return self.image.create(fromImage, tag).data @expose() def json(self, _id=None): From ca1a0f5cdbc08e3ab9c341610a15621516c88e3e Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Thu, 24 Mar 2016 15:05:50 +0530 Subject: [PATCH 12/62] Code merged --- dockyard/common/container/container.py | 7 ++---- dockyard/common/image/image.py | 2 +- dockyard/common/network/network.py | 26 +++++++++++++------ dockyard/common/utils.py | 6 ++--- dockyard/controllers/v1/network.py | 35 ++++++++++++++++++-------- 5 files changed, 50 insertions(+), 26 deletions(-) diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index c19f520..4c6de55 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -1,14 +1,11 @@ -from dockyard.common import base, link from dockyard.common import utils class Container(object): def __init__(self): - self.rest_client = base.RESTClient() + pass def list(self, name_id=None): - host = utils.get_host() - ln = link.make_url(host=host, protocol='http', url='/containers/json') - return self.rest_client.GET(ln).data + return utils.dispatch_get_request('/containers/json', 'http').data def stats(self, _id): return "stats" diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py index 33e27d3..9b6e71f 100644 --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -6,7 +6,7 @@ def __init__(self): self.rest_client = base.RESTClient() def list(self, _id=None): - return utils.dispatch_request(protocol='http', url='/images/json').data + return utils.dispatch_get_request(protocol='http', url='/images/json').data def create(self, fromImage, tag): url = '/images/create' diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py index b28a32f..d7f51bf 100644 --- a/dockyard/common/network/network.py +++ b/dockyard/common/network/network.py @@ -1,17 +1,29 @@ +<<<<<<< Updated upstream from dockyard.common import base, link +======= +import ast +from pecan import request + +>>>>>>> Stashed changes from dockyard.common import utils class Network(object): def __init__(self): - self.rest_client = base.RESTClient() + pass + + def list(self, name_or_id=None): + url = '/networks' + if name_or_id: + url = url + ('/%s' % (name_or_id)) + return utils.dispatch_get_request(url, 'http').data - def list(self, name_or_id): - host = utils.get_host() - ln = link.make_url(host=host, protocol='http', url='/networks/') - return str(self.rest_client.GET(ln).data) + def connect(self, _id): + url = ('/networks/%s/create' % (_id)) + body = ast.literal_eval(request.environ['webob._parsed_post_vars'][0].keys()[0]) + return utils.dispatch_post_request(url, 'http', body) - def set_network(self, _id, *connect): + def disconnect(self, _id): return "SET network" - def create(self, *args): + def create(self): return "Create Network" diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index 5ea8b77..2ab5625 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -16,10 +16,10 @@ def get_link(url, protocol): host = get_host() return link.make_url(host=host, protocol=protocol, url=url) -def dispatch_request(url, protocol): +def dispatch_get_request(url, protocol): ln = get_link(url, protocol) return rest_client.GET(ln) -def dispatch_post_request(url, protocol, query_params): +def dispatch_post_request(url, protocol, query_params=None, body=None): ln = get_link(url, protocol) - return rest_client.POST(url=ln, query_params=query_params) + return rest_client.POST(url=ln, query_params=query_params, body=body) diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py index 71b4082..6bdaa0a 100644 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -6,39 +6,54 @@ class Network(object): def __init__(self): self.network = network.Network() - @expose(generic=True) + @expose() def index(self, name_or_id=None): return self.network.list(name_or_id) - @index.when(method='POST') - def set_network(self, _id, *connect): - return self.network.set_network(_id, connect) + @expose(generic=True) + def connect(self): + abort(404) + + @connect.when(method='POST') + def _connect(self, _id): + return self.network.connect(_id) + + @expose(generic=True) + def disconnect(self): + abort(404) + + @disconnect.when(method='POST') + def _disconnect(self, _id): + return self.network.disconnect(_id) @expose(generic=True) def create(self): abort(404) @create.when(method="POST") - def create(self, *args): - return self.network.create(args) + def create(self): + return self.network.create() class NetworkController(object): def __init__(self): pass @expose() - def _lookup(self, id_name=None, op=None): + def _lookup(self, id_name_op=None, op=None): new_url = [] if op: - new_url.append([op]) + new_url.append(op) + elif id_name_op: + new_url.append(id_name_op) - if id_name: - new_url.append([id_name]) + if op: + new_url.append(id_name_op) if new_url: new_url = tuple(new_url) else: new_url = tuple(['']) + print new_url return Network(), new_url From 95642c810e26716ef3d1daef653d8126d668d150 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sun, 27 Mar 2016 21:44:49 +0530 Subject: [PATCH 13/62] Network and volume --- dockyard/common/base.py | 2 +- dockyard/common/container/container.py | 12 +++++--- dockyard/common/image/image.py | 4 +++ dockyard/common/network/network.py | 30 +++++++++++--------- dockyard/common/stubs.py | 11 ++++++++ dockyard/common/utils.py | 11 ++++++-- dockyard/common/volume/volume.py | 17 ++++++++---- dockyard/controllers/v1/container.py | 38 ++++++++++++++++---------- dockyard/controllers/v1/image.py | 8 ++++++ dockyard/controllers/v1/network.py | 19 +++++++------ dockyard/controllers/v1/volume.py | 20 ++++++++------ 11 files changed, 115 insertions(+), 57 deletions(-) create mode 100644 dockyard/common/stubs.py diff --git a/dockyard/common/base.py b/dockyard/common/base.py index 1881e27..0f9e04c 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -96,7 +96,7 @@ def request(self, method, url, query_params=None, headers=None, url += '?' + urlencode(query_params) if headers['Content-Type'] == 'application/json': response = self.agent(url).request(method, url, -# body=json.dumps(body), + body=json.dumps(body), headers=headers) if headers['Content-Type'] == 'application/x-www-form-urlencoded': response = self.agent(url).request(method, url, diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index 4c6de55..33bfed3 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -5,13 +5,16 @@ def __init__(self): pass def list(self, name_id=None): - return utils.dispatch_get_request('/containers/json', 'http').data + url = '/containers/json' + return utils.dispatch_get_request(url, 'http').data def stats(self, _id): - return "stats" + url = ('/containers/%s/stats' % (_id)) + return utils.dispatch_get_request(url, 'http') def archive(self, _id): - return "archive" + url = ('/containers/%s/archive' % (_id)) + return utils.dispatch_get_request(url, 'http') def upload(self, _id): return "upload" @@ -20,7 +23,8 @@ def copy(self, _id): return "copy" def logs(self, _id): - return "logs" + url = ('/containers/%s/logs' % (_id)) + return utils.dispatch_get_request(url, 'http') def start(self, _id): return "start" diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py index 9b6e71f..503b547 100644 --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -15,3 +15,7 @@ def create(self, fromImage, tag): def push(self, _id): return (("PUSH %s Image\n") % (_id)) + + def delete(self, _id): + url = ('/images/%s' % (_id)) + return utils.dispatch_delete_request(url=url, protocol='http') diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py index d7f51bf..f69ece4 100644 --- a/dockyard/common/network/network.py +++ b/dockyard/common/network/network.py @@ -1,10 +1,6 @@ -<<<<<<< Updated upstream -from dockyard.common import base, link -======= -import ast +import ast, json from pecan import request ->>>>>>> Stashed changes from dockyard.common import utils class Network(object): @@ -17,13 +13,21 @@ def list(self, name_or_id=None): url = url + ('/%s' % (name_or_id)) return utils.dispatch_get_request(url, 'http').data - def connect(self, _id): - url = ('/networks/%s/create' % (_id)) - body = ast.literal_eval(request.environ['webob._parsed_post_vars'][0].keys()[0]) - return utils.dispatch_post_request(url, 'http', body) + def connect(self, _id, data): + body = json.dumps(data) + url = ('/networks/%s/connect' % (_id)) + return utils.dispatch_post_request(url, 'http', body=body) - def disconnect(self, _id): - return "SET network" + def disconnect(self, _id, data): + body = json.dumps(data) + url = ('/networks/%s/disconnect' % (_id)) + return utils.dispatch_post_request(url, 'http', body=body) - def create(self): - return "Create Network" + def create(self, data): + url = ('/networks/create') + body = json.dumps(data) + return utils.dispatch_post_request(url, 'http', body=data) + + def delete(self, _id): + url = ('/networks/%s' % (_id)) + return utils.dispatch_delete_request(url, 'http').data diff --git a/dockyard/common/stubs.py b/dockyard/common/stubs.py new file mode 100644 index 0000000..b39de89 --- /dev/null +++ b/dockyard/common/stubs.py @@ -0,0 +1,11 @@ +import subprocess + +def send_request(url ,data): + headers = 'Content-Type: application/json' + command = "curl -H '%s' %s -d '%s'" % (headers, url, data) + print command + output = subprocess.check_output(command.split()) + #req = urllib2.Request(url, data, headers) + return str(output) + +#send_request('localhost:3333', {'aaaa'}) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index 2ab5625..d0c743b 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -20,6 +20,13 @@ def dispatch_get_request(url, protocol): ln = get_link(url, protocol) return rest_client.GET(ln) -def dispatch_post_request(url, protocol, query_params=None, body=None): +def dispatch_post_request(url, protocol, body=None, query_params=None): + from dockyard.common import stubs ln = get_link(url, protocol) - return rest_client.POST(url=ln, query_params=query_params, body=body) + print body + return stubs.send_request(ln, body) +# return rest_client.POST(url=ln, query_params=query_params, body=body) + +def dispatch_delete_request(url, protocol, query_params=None): + ln = get_link(url, protocol) + return rest_client.DELETE(ln) diff --git a/dockyard/common/volume/volume.py b/dockyard/common/volume/volume.py index edce2e6..39d238a 100644 --- a/dockyard/common/volume/volume.py +++ b/dockyard/common/volume/volume.py @@ -1,3 +1,4 @@ +import json from dockyard.common import base, link from dockyard.common import utils @@ -6,12 +7,16 @@ def __init__(self): self.rest_client = base.RESTClient() def list(self, name=None): - host = utils.get_host() - ln = link.make_url(host=host, protocol='http', url='/volumes') - return self.rest_client.GET(ln).data + url = '/volumes' + if name: + url = url + ('/%s' % (name)) + return utils.dispatch_get_request(url, 'http').data def delete(self, name): - return "Delete Volume" + url = ('/volumes/%s' % (name)) + return utils.dispatch_delete_request(url, 'http').data - def create(self): - return "POST create" + def create(self, data): + url = '/volumes/create' + body = json.dumps(data) + return utils.dispatch_post_request(url, 'http', body=body).data diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py index 02b14d4..60adfe5 100644 --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -1,7 +1,6 @@ from pecan import expose, abort -from dockyard.common.container import container - +from dockyard.common.container import container class Container(object): def __init__(self): self.container = container.Container() @@ -69,29 +68,40 @@ def rename(self): @rename.when(method="POST") def rename_POST(self, _id): return self.container.rename(_id) - class ContainerController(object): def __init__(self): pass - @expose(generic=True) - def create(self, *args): - abort(404) +# @expose(generic=True) +# def create(self, *args): +# abort(404) - @create.when(method="POST") - def create_container(self, *args): - return self.container.create(args) +# @create.when(method="POST") +# def create_container(self, *args): +# return self.container.create(args) @expose() def json(self): return Container().json() - def _lookup(self, _id=None, op=None): - if op != None: - new_url = (op, _id) + @expose() + def _lookup(self, id_name_op=None, op=None): + new_url = [] + print("Inside ++++++++++") + if op: + new_url.append(op) + elif id_name_op: + new_url.append(id_name_op) + + if op: + new_url.append(id_name_op) + + if new_url: + new_url = tuple(new_url) else: - new_url = tuple([_id]) + new_url = tuple(['']) - return Container(), new_url; + print(new_url) + return Container(), new_url diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py index b2b20e3..4f5526c 100644 --- a/dockyard/controllers/v1/image.py +++ b/dockyard/controllers/v1/image.py @@ -11,10 +11,18 @@ def __init__(self): def push(self, _id): return self.image.push(_id) + @expose(generic=True) + def index(self): + abort(404) + @expose(generic=True) def create(self, *args): abort(404) + @index.when(method="DELETE") + def delete(self, _id): + return self.image.delete(_id).data + @create.when(method="POST") def create_POST(self, fromImage, tag='latest'): return self.image.create(fromImage, tag).data diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py index 6bdaa0a..62d6e80 100644 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -6,33 +6,37 @@ class Network(object): def __init__(self): self.network = network.Network() - @expose() + @expose(generic=True) def index(self, name_or_id=None): return self.network.list(name_or_id) + @index.when(method="DELETE") + def index_delete(self, name_or_id): + return self.network.delete(name_or_id) + @expose(generic=True) def connect(self): abort(404) @connect.when(method='POST') - def _connect(self, _id): - return self.network.connect(_id) + def _connect(self, _id, **kwargs): + return self.network.connect(_id, kwargs) @expose(generic=True) def disconnect(self): abort(404) @disconnect.when(method='POST') - def _disconnect(self, _id): - return self.network.disconnect(_id) + def _disconnect(self, _id, **kwargs): + return self.network.disconnect(_id, kwargs) @expose(generic=True) def create(self): abort(404) @create.when(method="POST") - def create(self): - return self.network.create() + def _create(self, **kwargs): + return self.network.create(kwargs) class NetworkController(object): def __init__(self): @@ -55,5 +59,4 @@ def _lookup(self, id_name_op=None, op=None): else: new_url = tuple(['']) - print new_url return Network(), new_url diff --git a/dockyard/controllers/v1/volume.py b/dockyard/controllers/v1/volume.py index f450824..07ba179 100644 --- a/dockyard/controllers/v1/volume.py +++ b/dockyard/controllers/v1/volume.py @@ -8,33 +8,35 @@ def __init__(self): @expose(generic=True) def index(self, name=None): - return self.volume.list() + return self.volume.list(name) @index.when(method='DELETE') def index_DELETE(self, name): - return self.volume.delete() + return self.volume.delete(name) @expose(generic=True) def create(self): abort(404) @create.when(method='POST') - def create(self): - return self.volume.create() + def _create(self, **kwargs): + return self.volume.create(kwargs) class VolumeController(object): def __init__(self): pass @expose() - def _lookup(self, id_name=None, op=None): - new_url = [] + def _lookup(self, id_name_op=None, op=None): + new_url = [] if op: - new_url.append([op]) + new_url.append(op) + elif id_name_op: + new_url.append(id_name_op) - if id_name: - new_url.append([id_name]) + if op: + new_url.append(id_name_op) if new_url: new_url = tuple(new_url) From ed966e854fb895fd78d0d53879b6674f94474e99 Mon Sep 17 00:00:00 2001 From: karelyatin Date: Tue, 29 Mar 2016 19:28:05 +0530 Subject: [PATCH 14/62] used urllib for post --- dockyard/common/container/container.py | 6 ++++++ dockyard/common/link.py | 2 +- dockyard/common/stubs.py | 11 ----------- dockyard/common/utils.py | 16 ++++++++++++---- dockyard/controllers/v1/container.py | 16 +++++++--------- 5 files changed, 26 insertions(+), 25 deletions(-) delete mode 100644 dockyard/common/stubs.py diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index 33bfed3..88e970a 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -1,5 +1,7 @@ from dockyard.common import utils +import json + class Container(object): def __init__(self): pass @@ -16,6 +18,10 @@ def archive(self, _id): url = ('/containers/%s/archive' % (_id)) return utils.dispatch_get_request(url, 'http') + def create(self, body=None): + url = ('/containers/create') + return utils.dispatch_post_request(url, 'http', body=json.dumps(body)).data + def upload(self, _id): return "upload" diff --git a/dockyard/common/link.py b/dockyard/common/link.py index afc19e2..898ff35 100644 --- a/dockyard/common/link.py +++ b/dockyard/common/link.py @@ -12,7 +12,7 @@ def make_url(host='127.0.0.1', port=None, protocol='http', url=None): return url -def make_query_url(**kwargs): +def make_query_url(kwargs): url = '' for key, value in kwargs.iteritems(): url += key + '=' + value + '&' diff --git a/dockyard/common/stubs.py b/dockyard/common/stubs.py deleted file mode 100644 index b39de89..0000000 --- a/dockyard/common/stubs.py +++ /dev/null @@ -1,11 +0,0 @@ -import subprocess - -def send_request(url ,data): - headers = 'Content-Type: application/json' - command = "curl -H '%s' %s -d '%s'" % (headers, url, data) - print command - output = subprocess.check_output(command.split()) - #req = urllib2.Request(url, data, headers) - return str(output) - -#send_request('localhost:3333', {'aaaa'}) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index d0c743b..2a97196 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -3,6 +3,8 @@ from dockyard.common import base, link from dockyard.common.container.scheduler.round_robin import RoundRobinScheduler +from urllib3 import PoolManager + rest_client = base.RESTClient() def get_config(group, option): @@ -21,12 +23,18 @@ def dispatch_get_request(url, protocol): return rest_client.GET(ln) def dispatch_post_request(url, protocol, body=None, query_params=None): - from dockyard.common import stubs ln = get_link(url, protocol) - print body - return stubs.send_request(ln, body) -# return rest_client.POST(url=ln, query_params=query_params, body=body) + if query_params: + query=link.make_query_url(query_params) + ln = ln + '?' + query + return dispatch_post_req(url=ln, post_params=query_params, body=body) def dispatch_delete_request(url, protocol, query_params=None): ln = get_link(url, protocol) return rest_client.DELETE(ln) + +def dispatch_post_req(url, headers=None, body=None, post_params=None): + pool = PoolManager() + if not headers: + headers = {'Content-Type':'application/json'} + return pool.urlopen('POST', url, headers=headers, body=body) diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py index 60adfe5..3a2c264 100644 --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -74,22 +74,21 @@ class ContainerController(object): def __init__(self): pass -# @expose(generic=True) -# def create(self, *args): -# abort(404) + @expose(generic=True) + def create(self, *args): + abort(404) -# @create.when(method="POST") -# def create_container(self, *args): -# return self.container.create(args) + @create.when(method="POST") + def create_container(self, **args): + return container.Container().create(args) @expose() def json(self): - return Container().json() + return container.Container().json() @expose() def _lookup(self, id_name_op=None, op=None): new_url = [] - print("Inside ++++++++++") if op: new_url.append(op) elif id_name_op: @@ -103,5 +102,4 @@ def _lookup(self, id_name_op=None, op=None): else: new_url = tuple(['']) - print(new_url) return Container(), new_url From a5789832df4a7de66d663e9e0469e4fbcfc34994 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Tue, 29 Mar 2016 23:12:33 +0530 Subject: [PATCH 15/62] Container with post requests --- dockyard/common/container/container.py | 80 +++++++++++++++++-- dockyard/controllers/v1/container.py | 105 +++++++++++++++++++++++-- test/resize | 1 + test/update | 1 + 4 files changed, 174 insertions(+), 13 deletions(-) create mode 100644 test/resize create mode 100644 test/update diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index 88e970a..e2bc933 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -1,4 +1,4 @@ -from dockyard.common import utils +from dockyard.common import utils, link import json @@ -7,7 +7,31 @@ def __init__(self): pass def list(self, name_id=None): - url = '/containers/json' + if name_id: + url = ('/containers/%s/json' % (name_id)) + else: + url = '/containers/json' + return utils.dispatch_get_request(url, 'http').data + + def changes(self, name_id=None): + url = (('/containers/%s/changes') % (name_id)) + return utils.dispatch_get_request(url, 'http').data + + def resize(self, _id, **kwargs): + url = (('/containers/%s/changes') % (_id)) + + if not kwargs: + query = link.make_query_url(kwargs) + url = url +'?' + query + + return utils.dispatch_get_request(url, 'http').data + + def export(self, name_id=None): + url = (('/containers/%s/export') % (name_id)) + return utils.dispatch_get_request(url, 'http').data + + def top(self, name_id=None): + url = (('/containers/%s/top') % (name_id)) return utils.dispatch_get_request(url, 'http').data def stats(self, _id): @@ -16,7 +40,7 @@ def stats(self, _id): def archive(self, _id): url = ('/containers/%s/archive' % (_id)) - return utils.dispatch_get_request(url, 'http') + return utils.dispatch_post_request(url, 'http') def create(self, body=None): url = ('/containers/create') @@ -26,14 +50,28 @@ def upload(self, _id): return "upload" def copy(self, _id): - return "copy" + url = ('/containers/%s/copy' % (_id)) + return utils.dispatch_post_request(url, 'http').data def logs(self, _id): url = ('/containers/%s/logs' % (_id)) - return utils.dispatch_get_request(url, 'http') + return utils.dispatch_get_request(url, 'http').data def start(self, _id): - return "start" + url = ('/containers/%s/start' % (_id)) + return utils.dispatch_post_request(url, 'http').data + + def restart(self, _id): + url = ('/containers/%s/restart' % (_id)) + return utils.dispatch_post_request(url, 'http').data + + def kill(self, _id): + url = ('/containers/%s/kill' % (_id)) + return utils.dispatch_post_request(url, 'http').data + + def stop(self, _id): + url = ('/containers/%s/stop' % (_id)) + return utils.dispatch_post_request(url, 'http').data def exe(self, _id): return "exec" @@ -41,5 +79,31 @@ def exe(self, _id): def attach(self, _id): return "attach" - def rename(self, _id): - return "rename" + def rename(self, _id, query_params=None): + url = (('/containers/%s/rename') % (_id)) + + if query_params: + query = link.make_query_url(query_params) + url = url +'?' + query + + return utils.dispatch_post_request(url, 'http').data + + def update(self, _id, body=None): + url = ('/containers/%s/update' % (_id)) + return utils.dispatch_post_request(url, 'http', body=json.dumps(body)).data + + def pause(self, _id): + url = ('/containers/%s/pause' % (_id)) + return utils.dispatch_post_request(url, 'http').data + + def unpause(self, _id): + url = ('/containers/%s/unpause' % (_id)) + return utils.dispatch_post_request(url, 'http').data + + def wait(self, _id): + url = ('/containers/%s/wait' % (_id)) + return utils.dispatch_post_request(url, 'http').data + + def delete(self, _id): + url = ('/containers/%s' % (_id)) + return utils.dispatch_delete_request(url, 'http').data diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py index 3a2c264..1a4d991 100644 --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -1,6 +1,7 @@ from pecan import expose, abort from dockyard.common.container import container + class Container(object): def __init__(self): self.container = container.Container() @@ -9,6 +10,26 @@ def __init__(self): def index(self): return "index" + @index.when(method="DELETE") + def delete(self, _id): + return self.container.delete(_id) + + @expose(generic=True) + def resize(self, _id, **kwargs): + abort(404) + + @resize.when(method="POST") + def resize_(self, _id, **kwargs): + return self.container.resize(_id, **kwargs) + + @expose() + def changes(self, _id): + return self.container.changes(_id) + + @expose() + def export(self, _id): + return self.container.export(_id) + @expose() def json(self, name_id=None): return self.container.list(name_id) @@ -19,7 +40,7 @@ def stats(self, _id): @expose(generic=True) def archive(self, _id): - return self.container.archive(_id) + abort(404) @archive.when(method="PUT") def upload(self, _id): @@ -45,6 +66,30 @@ def start(self): def start_POST(self, _id): return self.container.start(_id) + @expose(generic=True) + def kill(self): + abort(404) + + @kill.when(method="POST") + def kill_POST(self, _id): + return self.container.kill(_id) + + @expose(generic=True) + def restart(self): + abort(404) + + @restart.when(method="POST") + def restart_POST(self, _id): + return self.container.restart(_id) + + @expose(generic=True) + def stop(self): + abort(404) + + @stop.when(method="POST") + def stop_POST(self, _id): + return self.container.stop(_id) + @expose(generic=True, route='exec') def exe(self): abort(404) @@ -66,8 +111,47 @@ def rename(self): abort(404) @rename.when(method="POST") - def rename_POST(self, _id): - return self.container.rename(_id) + def rename_POST(self, _id, **kwargs): + return self.container.rename(_id, kwargs) + + @expose() + def top(self, _id): + return self.container.top(_id) + + @expose(generic=True) + def update(self, _id): + abort(404) + + @update.when(method="POST") + def update_POST(self, _id, **kwargs): + return self.container.update(_id, kwargs) + + @expose(generic=True) + def pause(self, _id): + abort(404) + + @pause.when(method="POST") + def pause_POST(self, _id): + return self.container.pause(_id) + + @expose(generic=True) + def unpause(self, _id): + abort(404) + + @unpause.when(method="POST") + def unpause_POST(self, _id): + return self.container.unpause(_id) + + @expose(generic=True) + def wait(self, _id): + abort(404) + + @wait.when(method="POST") + def wait_POST(self, _id): + return self.container.wait(_id) + + def delete(self, _id): + return self.container.delete(_id) class ContainerController(object): @@ -84,10 +168,18 @@ def create_container(self, **args): @expose() def json(self): - return container.Container().json() + return Container().json() + + @expose(generic=True) + def index(self): + abort(404) + def index_DELETE(self, _id): + Container().delete() + + @expose() @expose() - def _lookup(self, id_name_op=None, op=None): + def _lookup(self, id_name_op=None, op=None, **kwargs): new_url = [] if op: new_url.append(op) @@ -97,6 +189,9 @@ def _lookup(self, id_name_op=None, op=None): if op: new_url.append(id_name_op) + if kwargs: + new_url.append(kwargs) + if new_url: new_url = tuple(new_url) else: diff --git a/test/resize b/test/resize new file mode 100644 index 0000000..b5726ae --- /dev/null +++ b/test/resize @@ -0,0 +1 @@ +curl localhost:5869/containers/$1/resize?h=40&w=80 diff --git a/test/update b/test/update new file mode 100644 index 0000000..07753ea --- /dev/null +++ b/test/update @@ -0,0 +1 @@ +curl -XPOST -H 'Content-Type: application/json' localhost:5869/containers/$1/update -d '{ "CpuShares": 512 }' From e632caa14c92c6e0193d893c255a5ad12d12f65e Mon Sep 17 00:00:00 2001 From: karelyatin Date: Tue, 29 Mar 2016 23:27:48 +0530 Subject: [PATCH 16/62] added methods network and volume --- .gitignore | 0 MANIFEST.in | 0 README.md | 0 dockyard/__init__.py | 0 dockyard/api/__init__.py | 0 dockyard/api/app.py | 0 dockyard/api/config.py | 0 dockyard/cmd/__init__.py | 0 dockyard/cmd/api.py | 0 dockyard/common/__init__.py | 0 dockyard/common/base.py | 0 dockyard/common/cluster/__init__.py | 0 dockyard/common/cluster/cluster.py | 0 dockyard/common/cluster/membership/__init__.py | 0 dockyard/common/container/__init__.py | 0 dockyard/common/container/container.py | 0 dockyard/common/container/scheduler/__init__.py | 0 dockyard/common/container/scheduler/base.py | 0 dockyard/common/container/scheduler/round_robin.py | 0 dockyard/common/image/__init__.py | 0 dockyard/common/image/image.py | 9 +++++++++ dockyard/common/information/__init__.py | 0 dockyard/common/information/information.py | 0 dockyard/common/link.py | 0 dockyard/common/network/__init__.py | 0 dockyard/common/network/network.py | 6 +++--- dockyard/common/utils.py | 5 ++++- dockyard/common/volume/__init__.py | 0 dockyard/common/volume/volume.py | 0 dockyard/controllers/__init__.py | 0 dockyard/controllers/root.py | 0 dockyard/controllers/v1/__init__.py | 0 dockyard/controllers/v1/cluster.py | 0 dockyard/controllers/v1/container.py | 0 dockyard/controllers/v1/image.py | 9 +++++++++ dockyard/controllers/v1/information.py | 0 dockyard/controllers/v1/network.py | 10 +++++----- dockyard/controllers/v1/volume.py | 11 ++--------- dockyard/model/__init__.py | 0 dockyard/tests/__init__.py | 0 dockyard/tests/config.py | 0 dockyard/tests/test_functional.py | 0 dockyard/tests/test_units.py | 0 setup.cfg | 0 setup.py | 0 45 files changed, 32 insertions(+), 18 deletions(-) mode change 100644 => 100755 .gitignore mode change 100644 => 100755 MANIFEST.in mode change 100644 => 100755 README.md mode change 100644 => 100755 dockyard/__init__.py mode change 100644 => 100755 dockyard/api/__init__.py mode change 100644 => 100755 dockyard/api/app.py mode change 100644 => 100755 dockyard/api/config.py mode change 100644 => 100755 dockyard/cmd/__init__.py mode change 100644 => 100755 dockyard/cmd/api.py mode change 100644 => 100755 dockyard/common/__init__.py mode change 100644 => 100755 dockyard/common/base.py mode change 100644 => 100755 dockyard/common/cluster/__init__.py mode change 100644 => 100755 dockyard/common/cluster/cluster.py mode change 100644 => 100755 dockyard/common/cluster/membership/__init__.py mode change 100644 => 100755 dockyard/common/container/__init__.py mode change 100644 => 100755 dockyard/common/container/container.py mode change 100644 => 100755 dockyard/common/container/scheduler/__init__.py mode change 100644 => 100755 dockyard/common/container/scheduler/base.py mode change 100644 => 100755 dockyard/common/container/scheduler/round_robin.py mode change 100644 => 100755 dockyard/common/image/__init__.py mode change 100644 => 100755 dockyard/common/image/image.py mode change 100644 => 100755 dockyard/common/information/__init__.py mode change 100644 => 100755 dockyard/common/information/information.py mode change 100644 => 100755 dockyard/common/link.py mode change 100644 => 100755 dockyard/common/network/__init__.py mode change 100644 => 100755 dockyard/common/network/network.py mode change 100644 => 100755 dockyard/common/utils.py mode change 100644 => 100755 dockyard/common/volume/__init__.py mode change 100644 => 100755 dockyard/common/volume/volume.py mode change 100644 => 100755 dockyard/controllers/__init__.py mode change 100644 => 100755 dockyard/controllers/root.py mode change 100644 => 100755 dockyard/controllers/v1/__init__.py mode change 100644 => 100755 dockyard/controllers/v1/cluster.py mode change 100644 => 100755 dockyard/controllers/v1/container.py mode change 100644 => 100755 dockyard/controllers/v1/image.py mode change 100644 => 100755 dockyard/controllers/v1/information.py mode change 100644 => 100755 dockyard/controllers/v1/network.py mode change 100644 => 100755 dockyard/controllers/v1/volume.py mode change 100644 => 100755 dockyard/model/__init__.py mode change 100644 => 100755 dockyard/tests/__init__.py mode change 100644 => 100755 dockyard/tests/config.py mode change 100644 => 100755 dockyard/tests/test_functional.py mode change 100644 => 100755 dockyard/tests/test_units.py mode change 100644 => 100755 setup.cfg mode change 100644 => 100755 setup.py diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/MANIFEST.in b/MANIFEST.in old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/dockyard/__init__.py b/dockyard/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/api/__init__.py b/dockyard/api/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/api/app.py b/dockyard/api/app.py old mode 100644 new mode 100755 diff --git a/dockyard/api/config.py b/dockyard/api/config.py old mode 100644 new mode 100755 diff --git a/dockyard/cmd/__init__.py b/dockyard/cmd/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/cmd/api.py b/dockyard/cmd/api.py old mode 100644 new mode 100755 diff --git a/dockyard/common/__init__.py b/dockyard/common/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/common/base.py b/dockyard/common/base.py old mode 100644 new mode 100755 diff --git a/dockyard/common/cluster/__init__.py b/dockyard/common/cluster/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/common/cluster/cluster.py b/dockyard/common/cluster/cluster.py old mode 100644 new mode 100755 diff --git a/dockyard/common/cluster/membership/__init__.py b/dockyard/common/cluster/membership/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/common/container/__init__.py b/dockyard/common/container/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py old mode 100644 new mode 100755 diff --git a/dockyard/common/container/scheduler/__init__.py b/dockyard/common/container/scheduler/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/common/container/scheduler/base.py b/dockyard/common/container/scheduler/base.py old mode 100644 new mode 100755 diff --git a/dockyard/common/container/scheduler/round_robin.py b/dockyard/common/container/scheduler/round_robin.py old mode 100644 new mode 100755 diff --git a/dockyard/common/image/__init__.py b/dockyard/common/image/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py old mode 100644 new mode 100755 index 503b547..7c24bdf --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -8,6 +8,15 @@ def __init__(self): def list(self, _id=None): return utils.dispatch_get_request(protocol='http', url='/images/json').data + def history(self, _id=None): + url='/images/%s/history' % _id + return utils.dispatch_get_request(protocol='http', url=url).data + + def search(self, term=None): + url='/images/search' + query = { "term" : term} + return utils.dispatch_get_request(protocol='http', url=url, query_params=query).data + def create(self, fromImage, tag): url = '/images/create' query = { "fromImage" : fromImage, "tag" : tag } diff --git a/dockyard/common/information/__init__.py b/dockyard/common/information/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/common/information/information.py b/dockyard/common/information/information.py old mode 100644 new mode 100755 diff --git a/dockyard/common/link.py b/dockyard/common/link.py old mode 100644 new mode 100755 diff --git a/dockyard/common/network/__init__.py b/dockyard/common/network/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py old mode 100644 new mode 100755 index f69ece4..913ad53 --- a/dockyard/common/network/network.py +++ b/dockyard/common/network/network.py @@ -16,17 +16,17 @@ def list(self, name_or_id=None): def connect(self, _id, data): body = json.dumps(data) url = ('/networks/%s/connect' % (_id)) - return utils.dispatch_post_request(url, 'http', body=body) + return utils.dispatch_post_request(url, 'http', body=body).data def disconnect(self, _id, data): body = json.dumps(data) url = ('/networks/%s/disconnect' % (_id)) - return utils.dispatch_post_request(url, 'http', body=body) + return utils.dispatch_post_request(url, 'http', body=body).data def create(self, data): url = ('/networks/create') body = json.dumps(data) - return utils.dispatch_post_request(url, 'http', body=data) + return utils.dispatch_post_request(url, 'http', body=body).data def delete(self, _id): url = ('/networks/%s' % (_id)) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py old mode 100644 new mode 100755 index 2a97196..e01dea6 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -18,8 +18,11 @@ def get_link(url, protocol): host = get_host() return link.make_url(host=host, protocol=protocol, url=url) -def dispatch_get_request(url, protocol): +def dispatch_get_request(url, protocol, query_params=None): ln = get_link(url, protocol) + if query_params: + query=link.make_query_url(query_params) + ln = ln + '?' + query return rest_client.GET(ln) def dispatch_post_request(url, protocol, body=None, query_params=None): diff --git a/dockyard/common/volume/__init__.py b/dockyard/common/volume/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/common/volume/volume.py b/dockyard/common/volume/volume.py old mode 100644 new mode 100755 diff --git a/dockyard/controllers/__init__.py b/dockyard/controllers/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/controllers/root.py b/dockyard/controllers/root.py old mode 100644 new mode 100755 diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/controllers/v1/cluster.py b/dockyard/controllers/v1/cluster.py old mode 100644 new mode 100755 diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py old mode 100644 new mode 100755 diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py old mode 100644 new mode 100755 index 4f5526c..3e45d07 --- a/dockyard/controllers/v1/image.py +++ b/dockyard/controllers/v1/image.py @@ -31,6 +31,15 @@ def create_POST(self, fromImage, tag='latest'): def json(self, _id=None): return self.image.list(_id) + @expose() + def search(self, term=None): + return self.image.search(term) + + @expose() + def history(self, _id=None): + return self.image.history(_id) + + class ImageController(object): def __init__(self): pass diff --git a/dockyard/controllers/v1/information.py b/dockyard/controllers/v1/information.py old mode 100644 new mode 100755 diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py old mode 100644 new mode 100755 index 62d6e80..1c17288 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -45,18 +45,18 @@ def __init__(self): @expose() def _lookup(self, id_name_op=None, op=None): new_url = [] - if op: new_url.append(op) - elif id_name_op: new_url.append(id_name_op) - - if op: + elif id_name_op != 'create': + new_url.append('') + new_url.append(id_name_op) + elif id_name_op == 'create': new_url.append(id_name_op) if new_url: new_url = tuple(new_url) else: - new_url = tuple(['']) + new_url = tuple(['']) return Network(), new_url diff --git a/dockyard/controllers/v1/volume.py b/dockyard/controllers/v1/volume.py old mode 100644 new mode 100755 index 07ba179..d9bb92e --- a/dockyard/controllers/v1/volume.py +++ b/dockyard/controllers/v1/volume.py @@ -27,20 +27,13 @@ def __init__(self): pass @expose() - def _lookup(self, id_name_op=None, op=None): - new_url = [] - + def _lookup(self, op=None): + new_url = [''] if op: new_url.append(op) - elif id_name_op: - new_url.append(id_name_op) - - if op: - new_url.append(id_name_op) if new_url: new_url = tuple(new_url) else: new_url = tuple(['']) - return Volume(), new_url diff --git a/dockyard/model/__init__.py b/dockyard/model/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/tests/__init__.py b/dockyard/tests/__init__.py old mode 100644 new mode 100755 diff --git a/dockyard/tests/config.py b/dockyard/tests/config.py old mode 100644 new mode 100755 diff --git a/dockyard/tests/test_functional.py b/dockyard/tests/test_functional.py old mode 100644 new mode 100755 diff --git a/dockyard/tests/test_units.py b/dockyard/tests/test_units.py old mode 100644 new mode 100755 diff --git a/setup.cfg b/setup.cfg old mode 100644 new mode 100755 diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 From 09b44bd609fd4dec03efb4094611081fdf444312 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sat, 2 Apr 2016 18:57:14 +0530 Subject: [PATCH 17/62] Changed permission and added some API --- .gitignore | 0 MANIFEST.in | 0 README.md | 0 bugs/fixed_bugs.txt | 3 + bugs/implemented_blueprints.txt | 3 + bugs/known_bugs.txt | 3 + bugs/new_blueprints.txt | 3 + dockyard/__init__.py | 0 dockyard/api/__init__.py | 0 dockyard/api/app.py | 0 dockyard/api/config.py | 0 dockyard/cmd/__init__.py | 0 dockyard/cmd/api.py | 0 dockyard/common/__init__.py | 0 dockyard/common/base.py | 0 dockyard/common/cluster/__init__.py | 0 dockyard/common/cluster/cluster.py | 0 .../common/cluster/membership/__init__.py | 0 dockyard/common/container/__init__.py | 0 dockyard/common/container/container.py | 65 ++++++++++++++----- .../common/container/scheduler/__init__.py | 0 dockyard/common/container/scheduler/base.py | 0 .../common/container/scheduler/round_robin.py | 0 dockyard/common/image/__init__.py | 0 dockyard/common/image/image.py | 0 dockyard/common/information/__init__.py | 0 dockyard/common/information/information.py | 0 dockyard/common/link.py | 0 dockyard/common/network/__init__.py | 0 dockyard/common/network/network.py | 0 dockyard/common/utils.py | 0 dockyard/common/volume/__init__.py | 0 dockyard/common/volume/volume.py | 0 dockyard/controllers/__init__.py | 0 dockyard/controllers/root.py | 0 dockyard/controllers/v1/__init__.py | 0 dockyard/controllers/v1/cluster.py | 0 dockyard/controllers/v1/container.py | 42 +++++++----- dockyard/controllers/v1/image.py | 0 dockyard/controllers/v1/information.py | 0 dockyard/controllers/v1/network.py | 0 dockyard/controllers/v1/volume.py | 0 dockyard/model/__init__.py | 0 dockyard/tests/__init__.py | 0 dockyard/tests/config.py | 0 dockyard/tests/test_functional.py | 0 dockyard/tests/test_units.py | 0 setup.cfg | 0 setup.py | 0 test/resize | 0 test/update | 0 51 files changed, 85 insertions(+), 34 deletions(-) mode change 100755 => 100644 .gitignore mode change 100755 => 100644 MANIFEST.in mode change 100755 => 100644 README.md create mode 100755 bugs/fixed_bugs.txt create mode 100755 bugs/implemented_blueprints.txt create mode 100755 bugs/known_bugs.txt create mode 100755 bugs/new_blueprints.txt mode change 100755 => 100644 dockyard/__init__.py mode change 100755 => 100644 dockyard/api/__init__.py mode change 100755 => 100644 dockyard/api/app.py mode change 100755 => 100644 dockyard/api/config.py mode change 100755 => 100644 dockyard/cmd/__init__.py mode change 100755 => 100644 dockyard/cmd/api.py mode change 100755 => 100644 dockyard/common/__init__.py mode change 100755 => 100644 dockyard/common/base.py mode change 100755 => 100644 dockyard/common/cluster/__init__.py mode change 100755 => 100644 dockyard/common/cluster/cluster.py mode change 100755 => 100644 dockyard/common/cluster/membership/__init__.py mode change 100755 => 100644 dockyard/common/container/__init__.py mode change 100755 => 100644 dockyard/common/container/container.py mode change 100755 => 100644 dockyard/common/container/scheduler/__init__.py mode change 100755 => 100644 dockyard/common/container/scheduler/base.py mode change 100755 => 100644 dockyard/common/container/scheduler/round_robin.py mode change 100755 => 100644 dockyard/common/image/__init__.py mode change 100755 => 100644 dockyard/common/image/image.py mode change 100755 => 100644 dockyard/common/information/__init__.py mode change 100755 => 100644 dockyard/common/information/information.py mode change 100755 => 100644 dockyard/common/link.py mode change 100755 => 100644 dockyard/common/network/__init__.py mode change 100755 => 100644 dockyard/common/network/network.py mode change 100755 => 100644 dockyard/common/utils.py mode change 100755 => 100644 dockyard/common/volume/__init__.py mode change 100755 => 100644 dockyard/common/volume/volume.py mode change 100755 => 100644 dockyard/controllers/__init__.py mode change 100755 => 100644 dockyard/controllers/root.py mode change 100755 => 100644 dockyard/controllers/v1/__init__.py mode change 100755 => 100644 dockyard/controllers/v1/cluster.py mode change 100755 => 100644 dockyard/controllers/v1/container.py mode change 100755 => 100644 dockyard/controllers/v1/image.py mode change 100755 => 100644 dockyard/controllers/v1/information.py mode change 100755 => 100644 dockyard/controllers/v1/network.py mode change 100755 => 100644 dockyard/controllers/v1/volume.py mode change 100755 => 100644 dockyard/model/__init__.py mode change 100755 => 100644 dockyard/tests/__init__.py mode change 100755 => 100644 dockyard/tests/config.py mode change 100755 => 100644 dockyard/tests/test_functional.py mode change 100755 => 100644 dockyard/tests/test_units.py mode change 100755 => 100644 setup.cfg mode change 100755 => 100644 setup.py mode change 100644 => 100755 test/resize mode change 100644 => 100755 test/update diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 diff --git a/MANIFEST.in b/MANIFEST.in old mode 100755 new mode 100644 diff --git a/README.md b/README.md old mode 100755 new mode 100644 diff --git a/bugs/fixed_bugs.txt b/bugs/fixed_bugs.txt new file mode 100755 index 0000000..2bf4cbb --- /dev/null +++ b/bugs/fixed_bugs.txt @@ -0,0 +1,3 @@ +================================================================= +Add fixed bugs in this file. Select a format to add bugs. +================================================================= diff --git a/bugs/implemented_blueprints.txt b/bugs/implemented_blueprints.txt new file mode 100755 index 0000000..a472e29 --- /dev/null +++ b/bugs/implemented_blueprints.txt @@ -0,0 +1,3 @@ +======================================================================== +Add already implemented. +======================================================================== diff --git a/bugs/known_bugs.txt b/bugs/known_bugs.txt new file mode 100755 index 0000000..6a7e934 --- /dev/null +++ b/bugs/known_bugs.txt @@ -0,0 +1,3 @@ +================================================================= +Add known bugs in this file. Select a format to add bugs. +================================================================= diff --git a/bugs/new_blueprints.txt b/bugs/new_blueprints.txt new file mode 100755 index 0000000..e7d6303 --- /dev/null +++ b/bugs/new_blueprints.txt @@ -0,0 +1,3 @@ +======================================================================== +Add blueprints that needs to implmented. +======================================================================== diff --git a/dockyard/__init__.py b/dockyard/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/api/__init__.py b/dockyard/api/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/api/app.py b/dockyard/api/app.py old mode 100755 new mode 100644 diff --git a/dockyard/api/config.py b/dockyard/api/config.py old mode 100755 new mode 100644 diff --git a/dockyard/cmd/__init__.py b/dockyard/cmd/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/cmd/api.py b/dockyard/cmd/api.py old mode 100755 new mode 100644 diff --git a/dockyard/common/__init__.py b/dockyard/common/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/common/base.py b/dockyard/common/base.py old mode 100755 new mode 100644 diff --git a/dockyard/common/cluster/__init__.py b/dockyard/common/cluster/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/common/cluster/cluster.py b/dockyard/common/cluster/cluster.py old mode 100755 new mode 100644 diff --git a/dockyard/common/cluster/membership/__init__.py b/dockyard/common/cluster/membership/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/common/container/__init__.py b/dockyard/common/container/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py old mode 100755 new mode 100644 index e2bc933..51e8698 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -6,11 +6,15 @@ class Container(object): def __init__(self): pass - def list(self, name_id=None): + def list(self, name_id=None, query_params=None): if name_id: url = ('/containers/%s/json' % (name_id)) else: url = '/containers/json' + + if query_params: + url = ('%s?%s' % (url, query_params)) + return utils.dispatch_get_request(url, 'http').data def changes(self, name_id=None): @@ -20,18 +24,21 @@ def changes(self, name_id=None): def resize(self, _id, **kwargs): url = (('/containers/%s/changes') % (_id)) - if not kwargs: + if kwargs: query = link.make_query_url(kwargs) - url = url +'?' + query + url = ('%s?%s' % (url, query )) return utils.dispatch_get_request(url, 'http').data def export(self, name_id=None): url = (('/containers/%s/export') % (name_id)) - return utils.dispatch_get_request(url, 'http').data + return utils.dispatch_get_request(url, 'http') - def top(self, name_id=None): + def top(self, name_id=None, query_params=None): url = (('/containers/%s/top') % (name_id)) + if query_params: + url = ('%s?%s' % (url, query_params)) + return utils.dispatch_get_request(url, 'http').data def stats(self, _id): @@ -53,24 +60,45 @@ def copy(self, _id): url = ('/containers/%s/copy' % (_id)) return utils.dispatch_post_request(url, 'http').data - def logs(self, _id): - url = ('/containers/%s/logs' % (_id)) + def logs(self, _id, query_string, **kwargs): + url = ('/containers/%s/logs?%s' % (_id, query_string)) + + if kwargs: + query = link.make_query_url(kwargs) + url = url +'/' + query + return utils.dispatch_get_request(url, 'http').data - def start(self, _id): + def start(self, _id, query_params=None): url = ('/containers/%s/start' % (_id)) + + if query_params: + url = ('%s?%s' % (url, query_params)) + return utils.dispatch_post_request(url, 'http').data - def restart(self, _id): + def restart(self, _id, query_params=None): url = ('/containers/%s/restart' % (_id)) + + if query_params: + url = ('%s?%s' % (url, query_params)) + return utils.dispatch_post_request(url, 'http').data - def kill(self, _id): + def kill(self, _id, query_params=None): url = ('/containers/%s/kill' % (_id)) + + if query_params: + url = ('%s?%s' % (url, query_params)) + return utils.dispatch_post_request(url, 'http').data - def stop(self, _id): + def stop(self, _id, query_params=None): url = ('/containers/%s/stop' % (_id)) + + if query_params: + url = ('%s?%s' % (url, query_params)) + return utils.dispatch_post_request(url, 'http').data def exe(self, _id): @@ -79,13 +107,13 @@ def exe(self, _id): def attach(self, _id): return "attach" - def rename(self, _id, query_params=None): + def rename(self, _id, **kwargs): url = (('/containers/%s/rename') % (_id)) - if query_params: - query = link.make_query_url(query_params) + if kwargs: + query = link.make_query_url(kwargs) url = url +'?' + query - + return utils.dispatch_post_request(url, 'http').data def update(self, _id, body=None): @@ -104,6 +132,11 @@ def wait(self, _id): url = ('/containers/%s/wait' % (_id)) return utils.dispatch_post_request(url, 'http').data - def delete(self, _id): + def delete(self, _id, **kwargs): url = ('/containers/%s' % (_id)) + + if kwargs: + query = link.make_query_url(kwargs) + url = ('%s?%s' % (url, query)) + return utils.dispatch_delete_request(url, 'http').data diff --git a/dockyard/common/container/scheduler/__init__.py b/dockyard/common/container/scheduler/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/common/container/scheduler/base.py b/dockyard/common/container/scheduler/base.py old mode 100755 new mode 100644 diff --git a/dockyard/common/container/scheduler/round_robin.py b/dockyard/common/container/scheduler/round_robin.py old mode 100755 new mode 100644 diff --git a/dockyard/common/image/__init__.py b/dockyard/common/image/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py old mode 100755 new mode 100644 diff --git a/dockyard/common/information/__init__.py b/dockyard/common/information/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/common/information/information.py b/dockyard/common/information/information.py old mode 100755 new mode 100644 diff --git a/dockyard/common/link.py b/dockyard/common/link.py old mode 100755 new mode 100644 diff --git a/dockyard/common/network/__init__.py b/dockyard/common/network/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py old mode 100755 new mode 100644 diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py old mode 100755 new mode 100644 diff --git a/dockyard/common/volume/__init__.py b/dockyard/common/volume/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/common/volume/volume.py b/dockyard/common/volume/volume.py old mode 100755 new mode 100644 diff --git a/dockyard/controllers/__init__.py b/dockyard/controllers/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/controllers/root.py b/dockyard/controllers/root.py old mode 100755 new mode 100644 diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/controllers/v1/cluster.py b/dockyard/controllers/v1/cluster.py old mode 100755 new mode 100644 diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py old mode 100755 new mode 100644 index 1a4d991..66faa1d --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -1,4 +1,4 @@ -from pecan import expose, abort +from pecan import expose, abort, request from dockyard.common.container import container @@ -8,11 +8,11 @@ def __init__(self): @expose(generic=True) def index(self): - return "index" + abort(404) @index.when(method="DELETE") - def delete(self, _id): - return self.container.delete(_id) + def delete(self, _id, **kwargs): + return self.container.delete(_id, **kwargs) @expose(generic=True) def resize(self, _id, **kwargs): @@ -20,7 +20,7 @@ def resize(self, _id, **kwargs): @resize.when(method="POST") def resize_(self, _id, **kwargs): - return self.container.resize(_id, **kwargs) + return self.container.resize(_id, **kwargs) @expose() def changes(self, _id): @@ -32,7 +32,8 @@ def export(self, _id): @expose() def json(self, name_id=None): - return self.container.list(name_id) + query_params = request.query_string + return self.container.list(name_id, query_params=query_params) @expose() def stats(self, _id): @@ -56,7 +57,8 @@ def copy_POST(self, _id): @expose(generic=True) def logs(self, _id): - return self.container.logs(_id) + query_params = request.query_string + return self.container.logs(_id, query_params) @expose(generic=True) def start(self): @@ -64,7 +66,8 @@ def start(self): @start.when(method="POST") def start_POST(self, _id): - return self.container.start(_id) + query_params = request.query_string + return self.container.start(_id, query_params=query_params) @expose(generic=True) def kill(self): @@ -72,7 +75,8 @@ def kill(self): @kill.when(method="POST") def kill_POST(self, _id): - return self.container.kill(_id) + query_params = request.query_string + return self.container.kill(_id, query_params=query_params) @expose(generic=True) def restart(self): @@ -88,7 +92,8 @@ def stop(self): @stop.when(method="POST") def stop_POST(self, _id): - return self.container.stop(_id) + query_params = request.query_string + return self.container.stop(_id, query_params=query_params) @expose(generic=True, route='exec') def exe(self): @@ -112,11 +117,12 @@ def rename(self): @rename.when(method="POST") def rename_POST(self, _id, **kwargs): - return self.container.rename(_id, kwargs) + return self.container.rename(_id, **kwargs) @expose() def top(self, _id): - return self.container.top(_id) + query_params = request.query_string + return self.container.top(_id, query_params=query_params) @expose(generic=True) def update(self, _id): @@ -174,12 +180,12 @@ def json(self): def index(self): abort(404) - def index_DELETE(self, _id): - Container().delete() - @expose() + def index_DELETE(self, _id, **kwargs): + Container().delete(_id, **kwargs) + @expose() - def _lookup(self, id_name_op=None, op=None, **kwargs): + def _lookup(self, id_name_op=None, op=None, *args): new_url = [] if op: new_url.append(op) @@ -189,8 +195,8 @@ def _lookup(self, id_name_op=None, op=None, **kwargs): if op: new_url.append(id_name_op) - if kwargs: - new_url.append(kwargs) + if args: + new_url.append(args) if new_url: new_url = tuple(new_url) diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py old mode 100755 new mode 100644 diff --git a/dockyard/controllers/v1/information.py b/dockyard/controllers/v1/information.py old mode 100755 new mode 100644 diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py old mode 100755 new mode 100644 diff --git a/dockyard/controllers/v1/volume.py b/dockyard/controllers/v1/volume.py old mode 100755 new mode 100644 diff --git a/dockyard/model/__init__.py b/dockyard/model/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/tests/__init__.py b/dockyard/tests/__init__.py old mode 100755 new mode 100644 diff --git a/dockyard/tests/config.py b/dockyard/tests/config.py old mode 100755 new mode 100644 diff --git a/dockyard/tests/test_functional.py b/dockyard/tests/test_functional.py old mode 100755 new mode 100644 diff --git a/dockyard/tests/test_units.py b/dockyard/tests/test_units.py old mode 100755 new mode 100644 diff --git a/setup.cfg b/setup.cfg old mode 100755 new mode 100644 diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 diff --git a/test/resize b/test/resize old mode 100644 new mode 100755 diff --git a/test/update b/test/update old mode 100644 new mode 100755 From 26297a87e74fbdf7c22ee284c1372650e1762e8c Mon Sep 17 00:00:00 2001 From: karelyatin Date: Sat, 2 Apr 2016 22:17:50 +0530 Subject: [PATCH 18/62] added methods to images --- dockyard/common/image/image.py | 4 ++++ dockyard/controllers/v1/image.py | 23 +++++++++++++++++++---- dockyard/controllers/v1/network.py | 15 +++++++-------- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py index 7c24bdf..4c7d259 100644 --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -28,3 +28,7 @@ def push(self, _id): def delete(self, _id): url = ('/images/%s' % (_id)) return utils.dispatch_delete_request(url=url, protocol='http') + + def tag(self, _id=None, kwargs): + url='/images/%s/tag' % _id + return utils.dispatch_post_request(protocol='http', url=url, query_params=kwargs).data diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py index 3e45d07..0bb81d3 100644 --- a/dockyard/controllers/v1/image.py +++ b/dockyard/controllers/v1/image.py @@ -19,14 +19,14 @@ def index(self): def create(self, *args): abort(404) - @index.when(method="DELETE") - def delete(self, _id): - return self.image.delete(_id).data - @create.when(method="POST") def create_POST(self, fromImage, tag='latest'): return self.image.create(fromImage, tag).data + @index.when(method="DELETE") + def delete(self, _id): + return self.image.delete(_id).data + @expose() def json(self, _id=None): return self.image.list(_id) @@ -39,6 +39,21 @@ def search(self, term=None): def history(self, _id=None): return self.image.history(_id) + @expose(generic=True) + def tag(self, *args): + abort(404) + + @tag.when(method="POST") + def tag_POST(self, id_or_name, **kwargs): + return self.image.tag(id_or_name, kwargs).data + + @expose(generic=True) + def build(self, *args): + abort(404) + + @build.when(method="POST") + def build_POST(self, **kwargs): + return self.image.build(kwargs).data class ImageController(object): def __init__(self): diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py index 1c17288..d6d7090 100644 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -30,17 +30,18 @@ def disconnect(self): def _disconnect(self, _id, **kwargs): return self.network.disconnect(_id, kwargs) + +class NetworkController(object): + def __init__(self): + self.network = network.Network() + @expose(generic=True) def create(self): abort(404) @create.when(method="POST") def _create(self, **kwargs): - return self.network.create(kwargs) - -class NetworkController(object): - def __init__(self): - pass + return self.network.create(kwargs) @expose() def _lookup(self, id_name_op=None, op=None): @@ -48,11 +49,9 @@ def _lookup(self, id_name_op=None, op=None): if op: new_url.append(op) new_url.append(id_name_op) - elif id_name_op != 'create': + elif id_name_op: new_url.append('') new_url.append(id_name_op) - elif id_name_op == 'create': - new_url.append(id_name_op) if new_url: new_url = tuple(new_url) From 215a1a563a2c6c9eb91c3b5c762f898946a5204b Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Mon, 4 Apr 2016 21:59:18 +0530 Subject: [PATCH 19/62] Changes for the latest api --- dockyard/common/base.py | 2 +- dockyard/common/container/container.py | 42 +++++++++++++++++++------- dockyard/common/image/image.py | 2 +- dockyard/common/utils.py | 25 +++++++++++++++ dockyard/controllers/v1/container.py | 12 +++++--- 5 files changed, 66 insertions(+), 17 deletions(-) diff --git a/dockyard/common/base.py b/dockyard/common/base.py index 0f9e04c..fe12efd 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -120,7 +120,7 @@ def request(self, method, url, query_params=None, headers=None, if sys.version_info > (3,): response.data = response.data.decode('utf8') # log response body - logger.debug("response body: %s" % response.data) + #logger.debug("response body: %s" % response.data) if response.status not in range(200, 206): raise ApiException(http_resp=response) diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index 51e8698..f603c3b 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -1,6 +1,7 @@ from dockyard.common import utils, link import json +from pecan.core import redirect class Container(object): def __init__(self): @@ -32,10 +33,12 @@ def resize(self, _id, **kwargs): def export(self, name_id=None): url = (('/containers/%s/export') % (name_id)) - return utils.dispatch_get_request(url, 'http') + redirect(utils.get_link(url, 'http')) + # return str(utils.dispatch_get_request(url, 'http').data) def top(self, name_id=None, query_params=None): url = (('/containers/%s/top') % (name_id)) + if query_params: url = ('%s?%s' % (url, query_params)) @@ -43,22 +46,33 @@ def top(self, name_id=None, query_params=None): def stats(self, _id): url = ('/containers/%s/stats' % (_id)) - return utils.dispatch_get_request(url, 'http') + redirect(utils.get_link(url, 'http')) + # return utils.dispatch_get_request(url, 'http') - def archive(self, _id): + def archive(self, _id, query_params=None): url = ('/containers/%s/archive' % (_id)) - return utils.dispatch_post_request(url, 'http') + + if query_params: + url = ("%s?%s" % (url, query_params)) + + #redirect(utils.get_link(url, 'http')) + return utils.dispatch_put_request(url, 'http').data def create(self, body=None): url = ('/containers/create') return utils.dispatch_post_request(url, 'http', body=json.dumps(body)).data - def upload(self, _id): - return "upload" + def upload(self, _id, body=None, query_params=None, **kwargs): + url = ('/containers/%s/archive' %(_id)) + if query_params: + url = ("%s?%s" % (url, query_params)) + + return utils.dispatch_put_request(url, 'http', body=body).data def copy(self, _id): url = ('/containers/%s/copy' % (_id)) - return utils.dispatch_post_request(url, 'http').data + redirect(utils.get_link(url, 'http')) + # return str(utils.dispatch_post_request(url, 'http').data) def logs(self, _id, query_string, **kwargs): url = ('/containers/%s/logs?%s' % (_id, query_string)) @@ -67,7 +81,8 @@ def logs(self, _id, query_string, **kwargs): query = link.make_query_url(kwargs) url = url +'/' + query - return utils.dispatch_get_request(url, 'http').data + redirect(utils.get_link(url, 'http')) + # return utils.dispatch_get_request(url, 'http').data def start(self, _id, query_params=None): url = ('/containers/%s/start' % (_id)) @@ -102,10 +117,15 @@ def stop(self, _id, query_params=None): return utils.dispatch_post_request(url, 'http').data def exe(self, _id): - return "exec" + abort(404) + + def attach(self, _id, query_params=None): + url = ('/containers/%s/attach' % (_id)) + + if query_params: + url = ('%s?%s' % (url, query_params)) - def attach(self, _id): - return "attach" + redirect(utils.get_link(url, 'http')) def rename(self, _id, **kwargs): url = (('/containers/%s/rename') % (_id)) diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py index 4c7d259..6fba7fd 100644 --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -29,6 +29,6 @@ def delete(self, _id): url = ('/images/%s' % (_id)) return utils.dispatch_delete_request(url=url, protocol='http') - def tag(self, _id=None, kwargs): + def tag(self, _id=None, **kwargs): url='/images/%s/tag' % _id return utils.dispatch_post_request(protocol='http', url=url, query_params=kwargs).data diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index e01dea6..e72d44b 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -11,33 +11,58 @@ def get_config(group, option): CONF = cfg.CONF return CONF.group.option + def get_host(): return RoundRobinScheduler().get_host() + def get_link(url, protocol): host = get_host() return link.make_url(host=host, protocol=protocol, url=url) + def dispatch_get_request(url, protocol, query_params=None): ln = get_link(url, protocol) + if query_params: query=link.make_query_url(query_params) ln = ln + '?' + query + return rest_client.GET(ln) + def dispatch_post_request(url, protocol, body=None, query_params=None): ln = get_link(url, protocol) + if query_params: query=link.make_query_url(query_params) ln = ln + '?' + query + return dispatch_post_req(url=ln, post_params=query_params, body=body) + +def dispatch_put_request(url, protocol, body=None, query_params=None): + ln = get_link(url, protocol) + if query_params: + query=link.make_query_url(query_params) + ln = ln + '?' + query + return dispatch_put_req(url=ln, post_params=query_params, body=body) + + def dispatch_delete_request(url, protocol, query_params=None): ln = get_link(url, protocol) return rest_client.DELETE(ln) + def dispatch_post_req(url, headers=None, body=None, post_params=None): pool = PoolManager() if not headers: headers = {'Content-Type':'application/json'} return pool.urlopen('POST', url, headers=headers, body=body) + + +def dispatch_put_req(url, headers=None, body=None, post_params=None): + pool = PoolManager() + if not headers: + headers = {'Content-Type':'application/x-tar'} + return pool.urlopen('PUT', url, headers=headers, body=body) diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py index 66faa1d..d9f08ab 100644 --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -41,11 +41,14 @@ def stats(self, _id): @expose(generic=True) def archive(self, _id): - abort(404) + query_params = request.query_string + return self.container.archive(_id, query_params) @archive.when(method="PUT") - def upload(self, _id): - return self.container.upload(_id) + def upload(self, _id, **kwargs): + body = request.body + query_params = request.query_string + return self.container.upload(_id, body=body, query_params=query_params) @expose(generic=True) def copy(self, _id): @@ -109,7 +112,8 @@ def attach(self): @attach.when(method="POST") def attach_POST(self, _id): - return self.container.attach(_id) + query_params = request.query_string + return self.container.attach(_id, query_params=query_params) @expose(generic=True) def rename(self): From 839f52c863bce3b5ca7eca5372759fe66ec8a3a0 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sat, 23 Apr 2016 13:41:02 -0400 Subject: [PATCH 20/62] Changes in containers, images, networks, no changes in volumes --- dockyard/api/app.py | 24 +-- dockyard/api/config.py | 20 +- dockyard/cmd/api.py | 7 +- dockyard/common/base.py | 4 +- dockyard/common/cluster/cluster.py | 3 +- dockyard/common/container/container.py | 185 +++++++----------- dockyard/common/container/scheduler/base.py | 2 +- .../common/container/scheduler/round_robin.py | 3 +- dockyard/common/image/image.py | 43 ++-- dockyard/common/information/information.py | 3 +- dockyard/common/link.py | 4 +- dockyard/common/network/network.py | 37 ++-- dockyard/common/url.py | 17 ++ dockyard/common/utils.py | 41 ++-- dockyard/common/volume/volume.py | 3 +- dockyard/controllers/root.py | 5 +- dockyard/controllers/v1/__init__.py | 3 +- dockyard/controllers/v1/cluster.py | 22 ++- dockyard/controllers/v1/container.py | 13 +- dockyard/controllers/v1/image.py | 8 +- dockyard/controllers/v1/information.py | 13 +- dockyard/controllers/v1/network.py | 11 +- dockyard/controllers/v1/volume.py | 14 +- etc/dockyard/dockyard.conf | 8 + requirement.txt | 1 + 25 files changed, 245 insertions(+), 249 deletions(-) create mode 100644 dockyard/common/url.py create mode 100644 etc/dockyard/dockyard.conf create mode 100644 requirement.txt diff --git a/dockyard/api/app.py b/dockyard/api/app.py index bb42e98..318e230 100644 --- a/dockyard/api/app.py +++ b/dockyard/api/app.py @@ -1,38 +1,40 @@ -import pecan - -from dockyard.api import config as api_config from oslo_config import cfg +import pecan from pecan import make_app -from dockyard import model +from dockyard.api import config as api_config +# To be used later for initialization of database +#from dockyard import model API_SERVICE_OPT = [ cfg.PortOpt('port', - default=5869, - help='Port for the dockyard service.'), + default=5869, + help='Port for the dockyard service.'), cfg.IPOpt('host', - default='127.0.0.1', + default='0.0.0.0', help='Listening address for dockyard service'), ] CONF = cfg.CONF opt_group = cfg.OptGroup(name='default', - title='Group for the default values for dockyard api') + title='Group for the default values of dockyard api') CONF.register_group(opt_group) CONF.register_opts(API_SERVICE_OPT, opt_group) + def get_pecan_config(): # Set up the pecan configuration filename = api_config.__file__.replace('.pyc', '.py') return pecan.configuration.conf_from_file(filename) -def setup_app(config=None): +def setup_app(config=None): if not config: - config = get_pecan_config() + config = get_pecan_config() - model.init_model() + # To be used later for initialization of database + # model.init_model() app_conf = dict(config.app) return make_app( diff --git a/dockyard/api/config.py b/dockyard/api/config.py index 7fa54d4..dabf7d8 100644 --- a/dockyard/api/config.py +++ b/dockyard/api/config.py @@ -2,23 +2,5 @@ app = { 'root': 'dockyard.controllers.root.RootController', 'modules': ['dockyard', 'dockyard.api'], - 'static_root': '%(confdir)s/public', - 'template_path': '%(confdir)s/dockyard/templates', - 'debug': True, - 'errors': { - 404: '/error/404', - '__force_dict__': True - }, - 'acl_public_routes': [ - '/', - '/v1', - ], - + 'debug': True } - -# Custom Configurations must be in Python dictionary format:: -# -# foo = {'bar':'baz'} -# -# All configurations are accessible at:: -# pecan.conf diff --git a/dockyard/cmd/api.py b/dockyard/cmd/api.py index 5d667b4..818df60 100644 --- a/dockyard/cmd/api.py +++ b/dockyard/cmd/api.py @@ -1,15 +1,16 @@ +from oslo_config import cfg from wsgiref import simple_server -from oslo_config import cfg -from oslo_log import log as logging from dockyard.api import app as api_app + def main(): app = api_app.setup_app() - + # create the wsgi server and start it host, port = cfg.CONF.default.host, cfg.CONF.default.port srv = simple_server.make_server(host, port, app) srv.serve_forever() + main() diff --git a/dockyard/common/base.py b/dockyard/common/base.py index fe12efd..44696c2 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -96,7 +96,7 @@ def request(self, method, url, query_params=None, headers=None, url += '?' + urlencode(query_params) if headers['Content-Type'] == 'application/json': response = self.agent(url).request(method, url, - body=json.dumps(body), + body=json.dumps(body), headers=headers) if headers['Content-Type'] == 'application/x-www-form-urlencoded': response = self.agent(url).request(method, url, @@ -192,6 +192,7 @@ def __str__(self): return error_message + class RESTClient(object): """ A class with methods to perform JSON requests. @@ -241,4 +242,3 @@ def DELETE(self, *n, **kw): Perform a DELETE request using `RESTClient.request()` """ return self.IMPL.DELETE(*n, **kw) - diff --git a/dockyard/common/cluster/cluster.py b/dockyard/common/cluster/cluster.py index 612cea6..b83764f 100644 --- a/dockyard/common/cluster/cluster.py +++ b/dockyard/common/cluster/cluster.py @@ -1,5 +1,6 @@ from oslo_config import cfg + class Cluster(object): def __init__(self): pass @@ -8,7 +9,7 @@ def register(self, cluster_id, host_ip, port): pass def unregister(self, cluster_id, host_ip): - pass + pass def get_hosts(self): return cfg.CONF.membership.hosts diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index f603c3b..d0353c0 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -1,162 +1,119 @@ from dockyard.common import utils, link +from dockyard.common import url import json from pecan.core import redirect -class Container(object): - def __init__(self): - pass - def list(self, name_id=None, query_params=None): - if name_id: - url = ('/containers/%s/json' % (name_id)) - else: - url = '/containers/json' +class Container(object): + base_url = '/containers' - if query_params: - url = ('%s?%s' % (url, query_params)) + def __init__(self): + self.url = url.URL(self.base_url) - return utils.dispatch_get_request(url, 'http').data + def list(self, name_or_id=None, query_params=None): + url_ = self.url.make_url(url_='json', id_=name_or_id) + return utils.dispatch_get_request(url_, query_params=query_params) - def changes(self, name_id=None): - url = (('/containers/%s/changes') % (name_id)) - return utils.dispatch_get_request(url, 'http').data + def changes(self, name_or_id=None): + url_ = self.url.make_url(url_='changes', id_=name_or_id) + return utils.dispatch_get_request(url_) - def resize(self, _id, **kwargs): - url = (('/containers/%s/changes') % (_id)) + def resize(self, id_, **kwargs): + url_ = self.url.make_url(url_='resize', id_=id_) if kwargs: query = link.make_query_url(kwargs) - url = ('%s?%s' % (url, query )) - return utils.dispatch_get_request(url, 'http').data + return utils.dispatch_get_request(url_, query_params=query) - def export(self, name_id=None): - url = (('/containers/%s/export') % (name_id)) - redirect(utils.get_link(url, 'http')) - # return str(utils.dispatch_get_request(url, 'http').data) + def export(self, name_or_id=None): + url_ = self.url.make_url(url_='export', id_=name_or_id) + redirect(utils.get_link(url_)) - def top(self, name_id=None, query_params=None): - url = (('/containers/%s/top') % (name_id)) + def top(self, name_or_id=None, query_params=None): + url_ = self.url.make_url(url_='top', id_=name_or_id) + return utils.dispatch_get_request(url_, query_params=query_params) - if query_params: - url = ('%s?%s' % (url, query_params)) + def stats(self, id_): + url_ = self.url.make_url(url_='stats', id_=id_) + redirect(utils.get_link(url_)) - return utils.dispatch_get_request(url, 'http').data - - def stats(self, _id): - url = ('/containers/%s/stats' % (_id)) - redirect(utils.get_link(url, 'http')) - # return utils.dispatch_get_request(url, 'http') - - def archive(self, _id, query_params=None): - url = ('/containers/%s/archive' % (_id)) - - if query_params: - url = ("%s?%s" % (url, query_params)) - - #redirect(utils.get_link(url, 'http')) - return utils.dispatch_put_request(url, 'http').data + def archive(self, id_, query_params=None): + url_ = self.url.make_url(url_='archive', id_=id_) + return utils.dispatch_put_request(url_, query_params=query_params) def create(self, body=None): - url = ('/containers/create') - return utils.dispatch_post_request(url, 'http', body=json.dumps(body)).data + url_ = self.url.make_url(url_='create') + return utils.dispatch_post_request(url_, body=json.dumps(body)) - def upload(self, _id, body=None, query_params=None, **kwargs): - url = ('/containers/%s/archive' %(_id)) - if query_params: - url = ("%s?%s" % (url, query_params)) - - return utils.dispatch_put_request(url, 'http', body=body).data + def upload(self, id_, body=None, query=None, **kwargs): + url_ = self.url.make_url(url_='archive', id_=id_) + return utils.dispatch_put_request(url_, body=body, query_params=query) - def copy(self, _id): - url = ('/containers/%s/copy' % (_id)) - redirect(utils.get_link(url, 'http')) - # return str(utils.dispatch_post_request(url, 'http').data) + def copy(self, id_): + url_ = self.url.make_url(url_='copy', id_=id_) + redirect(utils.get_link(url_)) - def logs(self, _id, query_string, **kwargs): - url = ('/containers/%s/logs?%s' % (_id, query_string)) + def logs(self, id_, query_string, **kwargs): + url_ = self.url.make_url(url_='logs', id_=id_) if kwargs: query = link.make_query_url(kwargs) - url = url +'/' + query - - redirect(utils.get_link(url, 'http')) - # return utils.dispatch_get_request(url, 'http').data - - def start(self, _id, query_params=None): - url = ('/containers/%s/start' % (_id)) - if query_params: - url = ('%s?%s' % (url, query_params)) + redirect(utils.get_link(url_, query_params=query)) - return utils.dispatch_post_request(url, 'http').data + def start(self, id_, query_params=None): + url_ = self.url.make_url(url_='start', id_=id_) + return utils.dispatch_post_request(url_, query_params=query_params) - def restart(self, _id, query_params=None): - url = ('/containers/%s/restart' % (_id)) + def restart(self, id_, query_params=None): + url_ = self.url.make_url(url_='restart', id_=id_) + return utils.dispatch_post_request(url_, query_params=query_params) - if query_params: - url = ('%s?%s' % (url, query_params)) + def kill(self, id_, query_params=None): + url_ = self.url.make_url(url_='kill', id_=id_) + return utils.dispatch_post_request(url_, query_params=query_params) - return utils.dispatch_post_request(url, 'http').data - - def kill(self, _id, query_params=None): - url = ('/containers/%s/kill' % (_id)) - - if query_params: - url = ('%s?%s' % (url, query_params)) - - return utils.dispatch_post_request(url, 'http').data - - def stop(self, _id, query_params=None): - url = ('/containers/%s/stop' % (_id)) - - if query_params: - url = ('%s?%s' % (url, query_params)) - - return utils.dispatch_post_request(url, 'http').data + def stop(self, id_, query_params=None): + url_ = self.url.make_url(url_='stop', id_=id_) + return utils.dispatch_post_request(url_, query_params=query_parmas) def exe(self, _id): abort(404) - def attach(self, _id, query_params=None): - url = ('/containers/%s/attach' % (_id)) + def attach(self, id_, query_params=None): + url_ = self.url.make_url(url_='attach', id_=id_) + redirect(utils.get_link(url_, quer_params=query_params)) - if query_params: - url = ('%s?%s' % (url, query_params)) + def rename(self, id_, **kwargs): + url_ = self.url.make_url(url_='rename', id_=id_) - redirect(utils.get_link(url, 'http')) - - def rename(self, _id, **kwargs): - url = (('/containers/%s/rename') % (_id)) - if kwargs: query = link.make_query_url(kwargs) - url = url +'?' + query - - return utils.dispatch_post_request(url, 'http').data - def update(self, _id, body=None): - url = ('/containers/%s/update' % (_id)) - return utils.dispatch_post_request(url, 'http', body=json.dumps(body)).data + return utils.dispatch_post_request(url_, query_params=query) + + def update(self, id_, body=None): + url_ = self.url.make_url(url_='update', id_=id_) + return utils.dispatch_post_request(url_, body=json.dumps(body)) - def pause(self, _id): - url = ('/containers/%s/pause' % (_id)) - return utils.dispatch_post_request(url, 'http').data + def pause(self, id_): + url_ = self.url.make_url(url_='pause', id_=id_) + return utils.dispatch_post_request(url_) - def unpause(self, _id): - url = ('/containers/%s/unpause' % (_id)) - return utils.dispatch_post_request(url, 'http').data + def unpause(self, id_): + url_ = self.url.make_url(url_='unpause', id_=id_) + return utils.dispatch_post_request(url_) - def wait(self, _id): - url = ('/containers/%s/wait' % (_id)) - return utils.dispatch_post_request(url, 'http').data + def wait(self, id_): + url_ = self.url.make_url(url_='wait', id_=id_) + return utils.dispatch_post_request(url_) - def delete(self, _id, **kwargs): - url = ('/containers/%s' % (_id)) + def delete(self, id_, **kwargs): + url_ = self.url.make_url(id_=id_) if kwargs: query = link.make_query_url(kwargs) - url = ('%s?%s' % (url, query)) - return utils.dispatch_delete_request(url, 'http').data + return utils.dispatch_delete_request(url_, query_params=query) diff --git a/dockyard/common/container/scheduler/base.py b/dockyard/common/container/scheduler/base.py index 7650b45..09d05f7 100644 --- a/dockyard/common/container/scheduler/base.py +++ b/dockyard/common/container/scheduler/base.py @@ -7,7 +7,7 @@ def get_host(self, *args, **kwargs): """This method returns host information to launch container. - It expects specification of docker container and + It expects specification of docker container and interact with membership managment protocol to find the host to run a container. diff --git a/dockyard/common/container/scheduler/round_robin.py b/dockyard/common/container/scheduler/round_robin.py index 1d2a0b2..89f9914 100644 --- a/dockyard/common/container/scheduler/round_robin.py +++ b/dockyard/common/container/scheduler/round_robin.py @@ -6,11 +6,12 @@ class RoundRobinScheduler(Scheduler): count = -1 + def __init__(self): self.cluster = cluster.Cluster() def get_host(self): hosts = self.cluster.get_hosts() - num_hosts = len(hosts) + num_hosts = len(hosts) self.count = (self.count + 1) % num_hosts return (hosts[self.count]) diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py index 6fba7fd..f451c95 100644 --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -1,34 +1,39 @@ +from pecan import abort from dockyard.common import base, link -from dockyard.common import utils +from dockyard.common import utils, url + class Image(object): + base_url = '/images' + def __init__(self): - self.rest_client = base.RESTClient() + self.url = url.URL(self.base_url) - def list(self, _id=None): - return utils.dispatch_get_request(protocol='http', url='/images/json').data + def list(self, id_=None): + url_ = self.url.make_url(url_='json', id_=id_) + return utils.dispatch_get_request(url=url_) - def history(self, _id=None): - url='/images/%s/history' % _id - return utils.dispatch_get_request(protocol='http', url=url).data + def history(self, id_=None): + url_ = self.url.make_url(url_='history', id_=id_) + return utils.dispatch_get_request(url=url_).data def search(self, term=None): - url='/images/search' - query = { "term" : term} - return utils.dispatch_get_request(protocol='http', url=url, query_params=query).data + url_ = self.url.make_url(url_='search') + query = {"term": term} + return utils.dispatch_get_request(url=url_, query_params=query) def create(self, fromImage, tag): - url = '/images/create' - query = { "fromImage" : fromImage, "tag" : tag } - return utils.dispatch_post_request(protocol='http', url=url, query_params=query) + url_ = self.url.make_url(url_='create') + query = {"fromImage": fromImage, "tag": tag} + return utils.dispatch_post_request(url=url_, query_params=query) def push(self, _id): - return (("PUSH %s Image\n") % (_id)) + abort(404) - def delete(self, _id): - url = ('/images/%s' % (_id)) - return utils.dispatch_delete_request(url=url, protocol='http') + def delete(self, id_): + url_ = self.url.make_url(id_=id_) + return utils.dispatch_delete_request(url=url_) def tag(self, _id=None, **kwargs): - url='/images/%s/tag' % _id - return utils.dispatch_post_request(protocol='http', url=url, query_params=kwargs).data + url_ = self.url.make_url(url_='tag', id_=id_) + return utils.dispatch_post_request(url=url_, query_params=kwargs) diff --git a/dockyard/common/information/information.py b/dockyard/common/information/information.py index 42a06d6..a5ddc36 100644 --- a/dockyard/common/information/information.py +++ b/dockyard/common/information/information.py @@ -1,6 +1,7 @@ from dockyard.common import base, link from dockyard.common import utils + class Information(object): def __init__(self): self.rest_client = base.RESTClient() @@ -10,6 +11,7 @@ def info(self): ln = link.make_url(host=host, protocol='http', url='/info') return self.rest_client.GET(ln).data + class Version(object): def __init__(self): self.rest_client = base.RESTClient() @@ -18,4 +20,3 @@ def version(self): host = utils.get_host() ln = link.make_url(host=host, protocol='http', url='/version') return self.rest_client.GET(ln).data - diff --git a/dockyard/common/link.py b/dockyard/common/link.py index 898ff35..e9d1e1a 100644 --- a/dockyard/common/link.py +++ b/dockyard/common/link.py @@ -1,8 +1,7 @@ - def make_url(host='127.0.0.1', port=None, protocol='http', url=None): head = protocol + '://' head = head + host - if port: + if port: head = head + ":" + port if url: @@ -12,6 +11,7 @@ def make_url(host='127.0.0.1', port=None, protocol='http', url=None): return url + def make_query_url(kwargs): url = '' for key, value in kwargs.iteritems(): diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py index 913ad53..36c9f0a 100644 --- a/dockyard/common/network/network.py +++ b/dockyard/common/network/network.py @@ -1,33 +1,34 @@ -import ast, json +import json from pecan import request -from dockyard.common import utils +from dockyard.common import url, utils + class Network(object): + base_url = '/networks' + def __init__(self): - pass + self.url = url.URL(self.base_url) def list(self, name_or_id=None): - url = '/networks' - if name_or_id: - url = url + ('/%s' % (name_or_id)) - return utils.dispatch_get_request(url, 'http').data + url_ = self.url.make_url(id_=name_or_id) + return utils.dispatch_get_request(url=url_) - def connect(self, _id, data): + def connect(self, id_, data): body = json.dumps(data) - url = ('/networks/%s/connect' % (_id)) - return utils.dispatch_post_request(url, 'http', body=body).data + url_ = self.url.make_url(url_='connect', id_=id_) + return utils.dispatch_post_request(url=url_, body=body) - def disconnect(self, _id, data): + def disconnect(self, id_, data): + url_ = self.url.make_url(url_='disconnect', id_=id_) body = json.dumps(data) - url = ('/networks/%s/disconnect' % (_id)) - return utils.dispatch_post_request(url, 'http', body=body).data + return utils.dispatch_post_request(url=url_, body=body) def create(self, data): - url = ('/networks/create') + url_ = self.url.make_url(url_='create') body = json.dumps(data) - return utils.dispatch_post_request(url, 'http', body=body).data + return utils.dispatch_post_request(url=url_, body=body) - def delete(self, _id): - url = ('/networks/%s' % (_id)) - return utils.dispatch_delete_request(url, 'http').data + def delete(self, id_): + url_ = self.url.make_url(id_=id_) + return utils.dispatch_delete_request(url=url_) diff --git a/dockyard/common/url.py b/dockyard/common/url.py new file mode 100644 index 0000000..adfa73e --- /dev/null +++ b/dockyard/common/url.py @@ -0,0 +1,17 @@ +class URL(object): + def __init__(self, base_url): + self.base_url = base_url + + def make_url(self, url_=None, id_=None): + """It takes id, url as input and makes complete url, first part + of url is fetched from object itself. + """ + url = self.base_url + + if id_: + url = (("%s/%s") % (url, id_)) + + if url_: + url = (("%s/%s") % (url, url_)) + + return url diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index e72d44b..00e169e 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -7,62 +7,69 @@ rest_client = base.RESTClient() + def get_config(group, option): CONF = cfg.CONF return CONF.group.option def get_host(): - return RoundRobinScheduler().get_host() + return RoundRobinScheduler().get_host() def get_link(url, protocol): host = get_host() return link.make_url(host=host, protocol=protocol, url=url) - -def dispatch_get_request(url, protocol, query_params=None): + +def dispatch_get_request(url, protocol='http', query_params=None): ln = get_link(url, protocol) + pool = PoolManager() if query_params: - query=link.make_query_url(query_params) + query = link.make_query_url(query_params) ln = ln + '?' + query - return rest_client.GET(ln) + #return rest_client.GET(ln).data + return pool.urlopen('GET', url=ln).data -def dispatch_post_request(url, protocol, body=None, query_params=None): +def dispatch_post_request(url, protocol='http', body=None, query_params=None): ln = get_link(url, protocol) if query_params: - query=link.make_query_url(query_params) + query = link.make_query_url(query_params) ln = ln + '?' + query - return dispatch_post_req(url=ln, post_params=query_params, body=body) + return dispatch_post_req(url=ln, post_params=query_params, body=body).data -def dispatch_put_request(url, protocol, body=None, query_params=None): +def dispatch_put_request(url, protocol='http', body=None, query_params=None): ln = get_link(url, protocol) + if query_params: - query=link.make_query_url(query_params) + query = link.make_query_url(query_params) ln = ln + '?' + query - return dispatch_put_req(url=ln, post_params=query_params, body=body) + + return dispatch_put_req(url=ln, post_params=query_params, body=body).data -def dispatch_delete_request(url, protocol, query_params=None): +def dispatch_delete_request(url, protocol='http', query_params=None): + pool = PoolManager() ln = get_link(url, protocol) - return rest_client.DELETE(ln) + return pool.urlopen('DELETE', url=ln) + #return rest_client.DELETE(ln).data def dispatch_post_req(url, headers=None, body=None, post_params=None): pool = PoolManager() if not headers: - headers = {'Content-Type':'application/json'} - return pool.urlopen('POST', url, headers=headers, body=body) + headers = {'Content-Type': 'application/json'} + return pool.urlopen('POST', url, headers=headers, body=body).data def dispatch_put_req(url, headers=None, body=None, post_params=None): pool = PoolManager() if not headers: - headers = {'Content-Type':'application/x-tar'} - return pool.urlopen('PUT', url, headers=headers, body=body) + headers = {'Content-Type': 'application/x-tar'} + return pool.urlopen('PUT', url, headers=headers, body=body).data diff --git a/dockyard/common/volume/volume.py b/dockyard/common/volume/volume.py index 39d238a..dbd49b5 100644 --- a/dockyard/common/volume/volume.py +++ b/dockyard/common/volume/volume.py @@ -2,6 +2,7 @@ from dockyard.common import base, link from dockyard.common import utils + class Volume(object): def __init__(self): self.rest_client = base.RESTClient() @@ -18,5 +19,5 @@ def delete(self, name): def create(self, data): url = '/volumes/create' - body = json.dumps(data) + body = json.dumps(data) return utils.dispatch_post_request(url, 'http', body=body).data diff --git a/dockyard/controllers/root.py b/dockyard/controllers/root.py index 8e96847..d88b9af 100644 --- a/dockyard/controllers/root.py +++ b/dockyard/controllers/root.py @@ -2,6 +2,7 @@ from dockyard.controllers import v1 + class RootController(rest.RestController): _version = ['v1'] @@ -18,7 +19,7 @@ def index(self): @expose() def _route(self, args): """Override default routing. - + It redirect to the default value of the dockyard API, if version of the API is not specified or wrongly specified """ @@ -26,4 +27,4 @@ def _route(self, args): if args[0] and args[0] not in self._version: args = [self._default_version] + args - return super(RootController, self)._route(args) + return super(RootController, self)._route(args) diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py index b92870a..0e7ab6f 100644 --- a/dockyard/controllers/v1/__init__.py +++ b/dockyard/controllers/v1/__init__.py @@ -7,11 +7,12 @@ from pecan import rest from pecan import expose + class Controller(rest.RestController): info = information.Information() version = information.Version() containers = container.ContainerController() images = image.ImageController() networks = network.NetworkController() - volumes = volume.VolumeController() + volumes = volume.VolumeController() clusters = cluster.ClusterController() diff --git a/dockyard/controllers/v1/cluster.py b/dockyard/controllers/v1/cluster.py index 826e3b6..4a7de57 100644 --- a/dockyard/controllers/v1/cluster.py +++ b/dockyard/controllers/v1/cluster.py @@ -3,6 +3,7 @@ from dockyard.common.cluster import cluster + class Cluster(object): def __init__(self): self.cluster = cluster.Cluster() @@ -14,7 +15,7 @@ def register(self, *args): @register.when(method="PUT") def register_POST(self, cluster_id, host_ip, port): return self.cluster.register(cluster_id, host_ip, port) - + @expose(generic=True) def unregister(self, *args): abort(404) @@ -23,25 +24,26 @@ def unregister(self, *args): def unregister_POST(self, cluster_id, host_ip): return self.cluster.unregister(cluster_id, host_ip) + class ClusterController(object): def __init__(self): HOST_SERVICE_OPT = [ cfg.ListOpt('hosts', - default='127.0.0.1:3333', - help='Listening address of docker service'), + default='127.0.0.1:3333', + help='Listening address of docker service'), ] CONF = cfg.CONF - opt_group = cfg.OptGroup(name='membership', - title='Group for membership of docker services') + opt_group = cfg.OptGroup(name='membership', title=("Group \ + for membership of docker services")) CONF.register_group(opt_group) CONF.register_opts(HOST_SERVICE_OPT, opt_group) @expose() def _lookup(self, op, cluster_id, host_ip, port=None): - new_url = [op, cluster_id, host_ip] + new_url = [op, cluster_id, host_ip] + + if port: + new_url.append(port) - if port: - new_url.append(port) - - return Cluster(), tuple(new_url) + return Cluster(), tuple(new_url) diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py index d9f08ab..b6d94bb 100644 --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -1,6 +1,7 @@ from pecan import expose, abort, request -from dockyard.common.container import container +from dockyard.common.container import container + class Container(object): def __init__(self): @@ -62,7 +63,7 @@ def copy_POST(self, _id): def logs(self, _id): query_params = request.query_string return self.container.logs(_id, query_params) - + @expose(generic=True) def start(self): abort(404) @@ -104,7 +105,7 @@ def exe(self): @exe.when(method="POST") def exec_POST(self, _id): - return self.container.exe(_id) + return self.container.exe(_id) @expose(generic=True) def attach(self): @@ -202,9 +203,9 @@ def _lookup(self, id_name_op=None, op=None, *args): if args: new_url.append(args) - if new_url: - new_url = tuple(new_url) + if new_url: + new_url = tuple(new_url) else: - new_url = tuple(['']) + new_url = tuple(['']) return Container(), new_url diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py index 0bb81d3..f9db886 100644 --- a/dockyard/controllers/v1/image.py +++ b/dockyard/controllers/v1/image.py @@ -2,12 +2,13 @@ from dockyard.common.image import image + class Image(object): def __init__(self): self.image = image.Image() super(Image, self).__init__() - @expose() + @expose() def push(self, _id): return self.image.push(_id) @@ -55,14 +56,15 @@ def build(self, *args): def build_POST(self, **kwargs): return self.image.build(kwargs).data + class ImageController(object): def __init__(self): pass @expose() def _lookup(self, _id, op=None): - if op != None: + if op is not None: new_url = (op, _id) else: new_url = tuple([_id]) - return Image(), new_url; + return Image(), new_url diff --git a/dockyard/controllers/v1/information.py b/dockyard/controllers/v1/information.py index 30bccca..20f1090 100644 --- a/dockyard/controllers/v1/information.py +++ b/dockyard/controllers/v1/information.py @@ -1,21 +1,22 @@ from pecan import expose -from pecan import rest from dockyard.common import base -from dockyard.common.information import information +from dockyard.common.information import information -class Information(rest.RestController): + +class Information(object): def __init__(self): self.information = information.Information() @expose() - def get(self): + def info(self): return self.information.info() -class Version(rest.RestController): + +class Version(object): def __init__(self): self.information = information.Version() @expose() - def get(self): + def version(self): return self.information.version() diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py index d6d7090..a8f786b 100644 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -2,6 +2,7 @@ from dockyard.common.network import network + class Network(object): def __init__(self): self.network = network.Network() @@ -30,7 +31,7 @@ def disconnect(self): def _disconnect(self, _id, **kwargs): return self.network.disconnect(_id, kwargs) - + class NetworkController(object): def __init__(self): self.network = network.Network() @@ -41,7 +42,7 @@ def create(self): @create.when(method="POST") def _create(self, **kwargs): - return self.network.create(kwargs) + return self.network.create(kwargs) @expose() def _lookup(self, id_name_op=None, op=None): @@ -53,9 +54,9 @@ def _lookup(self, id_name_op=None, op=None): new_url.append('') new_url.append(id_name_op) - if new_url: - new_url = tuple(new_url) + if new_url: + new_url = tuple(new_url) else: - new_url = tuple(['']) + new_url = tuple(['']) return Network(), new_url diff --git a/dockyard/controllers/v1/volume.py b/dockyard/controllers/v1/volume.py index d9bb92e..3cedf9d 100644 --- a/dockyard/controllers/v1/volume.py +++ b/dockyard/controllers/v1/volume.py @@ -1,7 +1,8 @@ -from pecan import expose +from pecan import expose from dockyard.common.volume import volume + class Volume(object): def __init__(self): self.volume = volume.Volume() @@ -13,7 +14,7 @@ def index(self, name=None): @index.when(method='DELETE') def index_DELETE(self, name): return self.volume.delete(name) - + @expose(generic=True) def create(self): abort(404) @@ -22,18 +23,19 @@ def create(self): def _create(self, **kwargs): return self.volume.create(kwargs) + class VolumeController(object): def __init__(self): pass @expose() def _lookup(self, op=None): - new_url = [''] + new_url = [''] if op: new_url.append(op) - if new_url: - new_url = tuple(new_url) + if new_url: + new_url = tuple(new_url) else: - new_url = tuple(['']) + new_url = tuple(['']) return Volume(), new_url diff --git a/etc/dockyard/dockyard.conf b/etc/dockyard/dockyard.conf new file mode 100644 index 0000000..9061c1e --- /dev/null +++ b/etc/dockyard/dockyard.conf @@ -0,0 +1,8 @@ +[default] +# host parameter defines the IP address on the server machine, to +# be used for binding to serve request. +host = 0.0.0.0 + +# port parameter defines the port on which dockyard server will keep +# on listening for REST APIs. +port = 5869 diff --git a/requirement.txt b/requirement.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/requirement.txt @@ -0,0 +1 @@ + From 1510445bffc28c555fd22139734efa104665acff Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sun, 24 Apr 2016 00:13:18 -0400 Subject: [PATCH 21/62] Changes for volumes --- dockyard/common/link.py | 9 ++++----- dockyard/common/volume/volume.py | 20 ++++++++++---------- requirement.txt | 4 +++- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/dockyard/common/link.py b/dockyard/common/link.py index e9d1e1a..7d1ac04 100644 --- a/dockyard/common/link.py +++ b/dockyard/common/link.py @@ -1,11 +1,10 @@ def make_url(host='127.0.0.1', port=None, protocol='http', url=None): - head = protocol + '://' - head = head + host + head = (("%s://%s") % (protocol, host)) if port: - head = head + ":" + port + head = (("%s:%s") % (head, port)) if url: - url = head + url + url = (('%s%s') % (head, url)) else: url = head @@ -15,6 +14,6 @@ def make_url(host='127.0.0.1', port=None, protocol='http', url=None): def make_query_url(kwargs): url = '' for key, value in kwargs.iteritems(): - url += key + '=' + value + '&' + url += (('%s=%s&') % (key, value)) url = url[:-1] return url diff --git a/dockyard/common/volume/volume.py b/dockyard/common/volume/volume.py index dbd49b5..432dd68 100644 --- a/dockyard/common/volume/volume.py +++ b/dockyard/common/volume/volume.py @@ -1,23 +1,23 @@ import json from dockyard.common import base, link -from dockyard.common import utils +from dockyard.common import utils, url class Volume(object): + base_url = '/volumes' + def __init__(self): - self.rest_client = base.RESTClient() + self.url = url.URL(self.base_url) def list(self, name=None): - url = '/volumes' - if name: - url = url + ('/%s' % (name)) - return utils.dispatch_get_request(url, 'http').data + url_ = self.url.make_url(id_=name) + return utils.dispatch_get_request(url_) def delete(self, name): - url = ('/volumes/%s' % (name)) - return utils.dispatch_delete_request(url, 'http').data + url_ = self.url.make_url(id_=name) + return utils.dispatch_delete_request(url_) def create(self, data): - url = '/volumes/create' + url_ = self.url.make_url(url_='create') body = json.dumps(data) - return utils.dispatch_post_request(url, 'http', body=body).data + return utils.dispatch_post_request(url, body=body) diff --git a/requirement.txt b/requirement.txt index 8b13789..e9eced6 100644 --- a/requirement.txt +++ b/requirement.txt @@ -1 +1,3 @@ - +oslo.log +oslo.config +pecan From 315687329739ac53568adecb7f6d8fd5f49aa3ff Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sun, 24 Apr 2016 04:23:49 -0400 Subject: [PATCH 22/62] Added functionality for driver of scheduler --- dockyard/common/information/information.py | 20 ++++++++------ dockyard/common/utils.py | 32 +++++++++++++++------- dockyard/controllers/v1/cluster.py | 2 +- dockyard/controllers/v1/container.py | 4 +-- dockyard/controllers/v1/information.py | 5 ++-- dockyard/controllers/v1/network.py | 4 +-- etc/dockyard/dockyard.conf | 4 +++ 7 files changed, 44 insertions(+), 27 deletions(-) diff --git a/dockyard/common/information/information.py b/dockyard/common/information/information.py index a5ddc36..063b9ac 100644 --- a/dockyard/common/information/information.py +++ b/dockyard/common/information/information.py @@ -1,22 +1,24 @@ from dockyard.common import base, link -from dockyard.common import utils +from dockyard.common import utils, url class Information(object): + base_url = '/info' + def __init__(self): - self.rest_client = base.RESTClient() + self.url = url.URL(self.base_url) def info(self): - host = utils.get_host() - ln = link.make_url(host=host, protocol='http', url='/info') - return self.rest_client.GET(ln).data + url_ = self.url.make_url() + return utils.dispatch_get_request(url_) class Version(object): + base_url = '/version' + def __init__(self): - self.rest_client = base.RESTClient() + self.url = url.URL(self.base_url) def version(self): - host = utils.get_host() - ln = link.make_url(host=host, protocol='http', url='/version') - return self.rest_client.GET(ln).data + url_ = self.url.make_url() + return utils.dispatch_get_request(url_) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index 00e169e..ab00717 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -1,11 +1,25 @@ +import importlib from oslo_config import cfg +from urllib3 import PoolManager from dockyard.common import base, link -from dockyard.common.container.scheduler.round_robin import RoundRobinScheduler -from urllib3 import PoolManager +SCHEDULER_OPT = [ + cfg.StrOpt('scheduler', + default='round_robin.RoundRobinScheduler', + help='Scheduler for the dockyard.'), +] + +CONF = cfg.CONF +CONF.register_opts(SCHEDULER_OPT, group='default') -rest_client = base.RESTClient() +# Fetch scheduler defined in the configuration file and load it. +scheduler_info = CONF.default.scheduler +scheduler_loc = 'dockyard.common.container.scheduler' +scheduler_info = (('%s.%s') % (scheduler_loc, scheduler_info)) +module_name, class_name = scheduler_info.rsplit(".", 1) +class_ = getattr(importlib.import_module(module_name), class_name) +scheduler = class_() def get_config(group, option): @@ -14,10 +28,10 @@ def get_config(group, option): def get_host(): - return RoundRobinScheduler().get_host() + return scheduler.get_host() -def get_link(url, protocol): +def get_link(url, protocol='http'): host = get_host() return link.make_url(host=host, protocol=protocol, url=url) @@ -28,9 +42,8 @@ def dispatch_get_request(url, protocol='http', query_params=None): if query_params: query = link.make_query_url(query_params) - ln = ln + '?' + query + ln = (('%s?%s') % (ln, query)) - #return rest_client.GET(ln).data return pool.urlopen('GET', url=ln).data @@ -39,7 +52,7 @@ def dispatch_post_request(url, protocol='http', body=None, query_params=None): if query_params: query = link.make_query_url(query_params) - ln = ln + '?' + query + ln = (('%s?%s') % (ln, query)) return dispatch_post_req(url=ln, post_params=query_params, body=body).data @@ -49,7 +62,7 @@ def dispatch_put_request(url, protocol='http', body=None, query_params=None): if query_params: query = link.make_query_url(query_params) - ln = ln + '?' + query + ln = (('%s?%s') % (ln, query)) return dispatch_put_req(url=ln, post_params=query_params, body=body).data @@ -58,7 +71,6 @@ def dispatch_delete_request(url, protocol='http', query_params=None): pool = PoolManager() ln = get_link(url, protocol) return pool.urlopen('DELETE', url=ln) - #return rest_client.DELETE(ln).data def dispatch_post_req(url, headers=None, body=None, post_params=None): diff --git a/dockyard/controllers/v1/cluster.py b/dockyard/controllers/v1/cluster.py index 4a7de57..677d5f3 100644 --- a/dockyard/controllers/v1/cluster.py +++ b/dockyard/controllers/v1/cluster.py @@ -21,7 +21,7 @@ def unregister(self, *args): abort(404) @unregister.when(method="DELETE") - def unregister_POST(self, cluster_id, host_ip): + def unregister_DELETE(self, cluster_id, host_ip): return self.cluster.unregister(cluster_id, host_ip) diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py index b6d94bb..8ac91f9 100644 --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -20,7 +20,7 @@ def resize(self, _id, **kwargs): abort(404) @resize.when(method="POST") - def resize_(self, _id, **kwargs): + def resize_POST(self, _id, **kwargs): return self.container.resize(_id, **kwargs) @expose() @@ -59,7 +59,7 @@ def copy(self, _id): def copy_POST(self, _id): return self.container.copy(_id) - @expose(generic=True) + @expose() def logs(self, _id): query_params = request.query_string return self.container.logs(_id, query_params) diff --git a/dockyard/controllers/v1/information.py b/dockyard/controllers/v1/information.py index 20f1090..a3030c3 100644 --- a/dockyard/controllers/v1/information.py +++ b/dockyard/controllers/v1/information.py @@ -1,6 +1,5 @@ from pecan import expose -from dockyard.common import base from dockyard.common.information import information @@ -9,7 +8,7 @@ def __init__(self): self.information = information.Information() @expose() - def info(self): + def index(self): return self.information.info() @@ -18,5 +17,5 @@ def __init__(self): self.information = information.Version() @expose() - def version(self): + def index(self): return self.information.version() diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py index a8f786b..1fab92c 100644 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -20,7 +20,7 @@ def connect(self): abort(404) @connect.when(method='POST') - def _connect(self, _id, **kwargs): + def connect_(self, _id, **kwargs): return self.network.connect(_id, kwargs) @expose(generic=True) @@ -28,7 +28,7 @@ def disconnect(self): abort(404) @disconnect.when(method='POST') - def _disconnect(self, _id, **kwargs): + def disconnect_(self, _id, **kwargs): return self.network.disconnect(_id, kwargs) diff --git a/etc/dockyard/dockyard.conf b/etc/dockyard/dockyard.conf index 9061c1e..99c8d77 100644 --- a/etc/dockyard/dockyard.conf +++ b/etc/dockyard/dockyard.conf @@ -6,3 +6,7 @@ host = 0.0.0.0 # port parameter defines the port on which dockyard server will keep # on listening for REST APIs. port = 5869 + +# Scheduler to be used for launching the containers and for other load +# task. +scheduler = round_robin.RoundRobinScheduler From ba00f2f8901b0734e07412f06fd6d678bc046513 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sun, 24 Apr 2016 04:26:19 -0400 Subject: [PATCH 23/62] Removed base.py --- dockyard/common/base.py | 244 ---------------------------------------- 1 file changed, 244 deletions(-) delete mode 100644 dockyard/common/base.py diff --git a/dockyard/common/base.py b/dockyard/common/base.py deleted file mode 100644 index 44696c2..0000000 --- a/dockyard/common/base.py +++ /dev/null @@ -1,244 +0,0 @@ -import sys -import io -import json - -from oslo_log import log as logging - -# python 2 and python 3 compatibility library -from six import iteritems - -try: - import urllib3 -except ImportError: - raise ImportError('python client requires urllib3') - -try: - # for python3 - from urllib.parse import urlencode -except ImportError: - # for python2 - from urllib import urlencode - - -logger = logging.getLogger(__name__) - - -class RESTResponse(io.IOBase): - - def __init__(self, resp): - self.urllib3_response = resp - self.status = resp.status - self.reason = resp.reason - self.data = resp.data - - def getheaders(self): - """ - Returns a dictionary of the response headers. - """ - return self.urllib3_response.getheaders() - - def getheader(self, name, default=None): - """ - Returns a given response header. - """ - return self.urllib3_response.getheader(name, default) - - -class RESTClientObject(object): - - def __init__(self, pools_size=4): - # http pool manager - self.pool_manager = urllib3.PoolManager( - num_pools=pools_size - ) - - def agent(self, url): - """ - Use `urllib3.util.parse_url` for backward compatibility. - Return proper pool manager for the http/https schemes. - """ - url = urllib3.util.parse_url(url) - scheme = url.scheme - if scheme == 'https': - return self.ssl_pool_manager - else: - return self.pool_manager - - def request(self, method, url, query_params=None, headers=None, - body=None, post_params=None): - """ - :param method: http request method - :param url: http request url - :param query_params: query parameters in the url - :param headers: http request headers - :param body: request json body, for `application/json` - :param post_params: request post parameters, - `application/x-www-form-urlencode` - and `multipart/form-data` - """ - method = method.upper() - assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH'] - - if post_params and body: - raise ValueError( - "body parameter cannot be used with post_params parameter." - ) - - post_params = post_params or {} - headers = headers or {} - - if 'Content-Type' not in headers: - headers['Content-Type'] = 'application/json' - - # For `POST`, `PUT`, `PATCH` - if method in ['POST', 'PUT', 'PATCH']: - if query_params: - url += '?' + urlencode(query_params) - if headers['Content-Type'] == 'application/json': - response = self.agent(url).request(method, url, - body=json.dumps(body), - headers=headers) - if headers['Content-Type'] == 'application/x-www-form-urlencoded': - response = self.agent(url).request(method, url, - encode_multipart=False, - headers=headers) - if headers['Content-Type'] == 'multipart/form-data': - # must del headers['Content-Type'], or the correct Content-Type - # which generated by urllib3 will be overwritten. - del headers['Content-Type'] - response = self.agent(url).request(method, url, - encode_multipart=True, - headers=headers) - # For `GET`, `HEAD`, `DELETE` - else: - response = self.agent(url).request(method, url, - headers=headers) - response = RESTResponse(response) - - # In the python 3, the response.data is bytes. - # we need to decode it to string. - if sys.version_info > (3,): - response.data = response.data.decode('utf8') - # log response body - #logger.debug("response body: %s" % response.data) - - if response.status not in range(200, 206): - raise ApiException(http_resp=response) - - return response - - def GET(self, url, headers=None, query_params=None): - return self.request("GET", url, - headers=headers, - query_params=query_params) - - def HEAD(self, url, headers=None, query_params=None): - return self.request("HEAD", url, - headers=headers, - query_params=query_params) - - def DELETE(self, url, headers=None, query_params=None): - return self.request("DELETE", url, - headers=headers, - query_params=query_params) - - def POST(self, url, headers=None, query_params=None, post_params=None, body=None): - return self.request("POST", url, - headers=headers, - query_params=query_params, - post_params=post_params, - body=body) - - def PUT(self, url, headers=None, query_params=None, post_params=None, body=None): - return self.request("PUT", url, - headers=headers, - query_params=query_params, - post_params=post_params, - body=body) - - def PATCH(self, url, headers=None, query_params=None, post_params=None, body=None): - return self.request("PATCH", url, - headers=headers, - query_params=query_params, - post_params=post_params, - body=body) - - -class ApiException(Exception): - - def __init__(self, status=None, reason=None, http_resp=None): - if http_resp: - self.status = http_resp.status - self.reason = http_resp.reason - self.body = http_resp.data - self.headers = http_resp.getheaders() - else: - self.status = status - self.reason = reason - self.body = None - self.headers = None - - def __str__(self): - """ - Custom error messages for exception - """ - error_message = "({0})\n"\ - "Reason: {1}\n".format(self.status, self.reason) - if self.headers: - error_message += "HTTP response headers: {0}\n".format(self.headers) - - if self.body: - error_message += "HTTP response body: {0}\n".format(self.body) - - return error_message - - -class RESTClient(object): - """ - A class with methods to perform JSON requests. - """ - - def __init__(self): - self.IMPL = RESTClientObject() - - def request(self, *n, **kw): - """ - Perform a REST request and parse the response. - """ - return self.IMPL.request(*n, **kw) - - def GET(self, *n, **kw): - """ - Perform a GET request using `RESTClient.request()`. - """ - return self.IMPL.GET(*n, **kw) - - def HEAD(self, *n, **kw): - """ - Perform a HEAD request using `RESTClient.request()`. - """ - return self.IMPL.GET(*n, **kw) - - def POST(self, *n, **kw): - """ - Perform a POST request using `RESTClient.request()` - """ - return self.IMPL.POST(*n, **kw) - - def PUT(self, *n, **kw): - """ - Perform a PUT request using `RESTClient.request()` - """ - return self.IMPL.PUT(*n, **kw) - - def PATCH(self, *n, **kw): - """ - Perform a PATCH request using `RESTClient.request()` - """ - return self.IMPL.PATCH(*n, **kw) - - def DELETE(self, *n, **kw): - """ - Perform a DELETE request using `RESTClient.request()` - """ - return self.IMPL.DELETE(*n, **kw) From 89591502385e0e9101f78d1b7d610152dd7279b8 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Mon, 25 Apr 2016 09:28:48 -0400 Subject: [PATCH 24/62] Changes for membership --- dockyard/common/membership/__init__.py | 0 dockyard/common/membership/base.py | 9 +++++++++ dockyard/common/membership/consul.py | 13 +++++++++++++ requirement.txt | 1 + 4 files changed, 23 insertions(+) create mode 100644 dockyard/common/membership/__init__.py create mode 100644 dockyard/common/membership/base.py create mode 100644 dockyard/common/membership/consul.py diff --git a/dockyard/common/membership/__init__.py b/dockyard/common/membership/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/common/membership/base.py b/dockyard/common/membership/base.py new file mode 100644 index 0000000..a0aa95d --- /dev/null +++ b/dockyard/common/membership/base.py @@ -0,0 +1,9 @@ +import abc + + +class Membership(object): + @abc.abstractmethod + def get_all_host(self): + """This method returns all the hosts available in the + data center. + """ diff --git a/dockyard/common/membership/consul.py b/dockyard/common/membership/consul.py new file mode 100644 index 0000000..fe82b62 --- /dev/null +++ b/dockyard/common/membership/consul.py @@ -0,0 +1,13 @@ +from consul.Consul import Catalog + +from dockyard.common.membership.base import Membership + +class Consul(Membership): + def __init__(self): + self.catalog = Catalog() + + def get_all_host(self): + """Returns all the members current agent sees. + """ + nodes = self.catalog.nodes() + return nodes diff --git a/requirement.txt b/requirement.txt index e9eced6..23e55f0 100644 --- a/requirement.txt +++ b/requirement.txt @@ -1,3 +1,4 @@ oslo.log oslo.config pecan +python-consul From 0eff03a06a223362c61aa6901e40e3693d46615b Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Mon, 25 Apr 2016 22:17:10 +0530 Subject: [PATCH 25/62] Changes for consul --- dockyard/common/membership/consul.py | 13 ------- dockyard/common/membership/consul_driver.py | 42 +++++++++++++++++++++ dockyard/common/utils.py | 21 +++++++++++ etc/dockyard/dockyard.conf | 11 ++++++ 4 files changed, 74 insertions(+), 13 deletions(-) delete mode 100644 dockyard/common/membership/consul.py create mode 100644 dockyard/common/membership/consul_driver.py diff --git a/dockyard/common/membership/consul.py b/dockyard/common/membership/consul.py deleted file mode 100644 index fe82b62..0000000 --- a/dockyard/common/membership/consul.py +++ /dev/null @@ -1,13 +0,0 @@ -from consul.Consul import Catalog - -from dockyard.common.membership.base import Membership - -class Consul(Membership): - def __init__(self): - self.catalog = Catalog() - - def get_all_host(self): - """Returns all the members current agent sees. - """ - nodes = self.catalog.nodes() - return nodes diff --git a/dockyard/common/membership/consul_driver.py b/dockyard/common/membership/consul_driver.py new file mode 100644 index 0000000..52ed480 --- /dev/null +++ b/dockyard/common/membership/consul_driver.py @@ -0,0 +1,42 @@ +import consul +from oslo_config import cfg + +from dockyard.common.membership.base import Membership + + +CONSUL_SERVICE_OPT = [ + cfg.StrOpt('service_name', + default='dockyard', + help='Name of this service used by the consul'), +] + +CONF = cfg.CONF +opt_group = cfg.OptGroup(name='consul', + title='Group for the consul parameters') +CONF.register_group(opt_group) +CONF.register_opts(CONSUL_SERVICE_OPT, opt_group) + +class Consul(Membership): + def __init__(self): +# print help(consul.Consul.Agent.Service.__init__) + self.service = consul.Consul.Agent.Service(consul.Consul.Agent()) + self.catalog = consul.Consul.Catalog() + + def register(self): + ip = CONF['default']['host'] + if ip == '0.0.0.0': + ip = None + + self.service.register(name=CONF.consul.service_name, + address = ip, + port = CONF.default.port, + tags = CONF.default.agent) + + def _get_services(self, services): + return services[1] + + def get_all_host(self, tag='agent'): + """Returns all the members current agent sees. + """ + services = self.catalog.service(CONF.consul.service_name, tag=tag) + return self._get_services(services) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index ab00717..bdfdd2f 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -15,6 +15,7 @@ # Fetch scheduler defined in the configuration file and load it. scheduler_info = CONF.default.scheduler +# May be this path can be specified in the configuration file. scheduler_loc = 'dockyard.common.container.scheduler' scheduler_info = (('%s.%s') % (scheduler_loc, scheduler_info)) module_name, class_name = scheduler_info.rsplit(".", 1) @@ -22,12 +23,32 @@ scheduler = class_() +MEMBERSHIP_OPT = [ + cfg.StrOpt('membership', + default='consul_driver.Consul', + help='Scheduler for the dockyard.'), +] + +CONF.register_opts(MEMBERSHIP_OPT, group='default') + +# Fetch scheduler defined in the configuration file and load it. +membership_info = CONF.default.membership +# May be this path can be specified in the configuration file. +membership_loc = 'dockyard.common.membership' +membership_info = (('%s.%s') % (membership_loc, membership_info)) +module_name, class_name = membership_info.rsplit(".", 1) +class_ = getattr(importlib.import_module(module_name), class_name) +membership = class_() +membership.register() + + def get_config(group, option): CONF = cfg.CONF return CONF.group.option def get_host(): +# print membership.get_all_hosts() return scheduler.get_host() diff --git a/etc/dockyard/dockyard.conf b/etc/dockyard/dockyard.conf index 99c8d77..4b8c9b5 100644 --- a/etc/dockyard/dockyard.conf +++ b/etc/dockyard/dockyard.conf @@ -10,3 +10,14 @@ port = 5869 # Scheduler to be used for launching the containers and for other load # task. scheduler = round_robin.RoundRobinScheduler + +# For managment of the members, we must use some kind of membership management +# tool. There are support for consul, ..... . +membership = consul_driver.Consul + +# Type of the agent, it is based on the role being taken by the package +# This product can behave differently based on the role being given by it. +agent = master + +[consul] +service_name = 'dockyard' From aa263611991f717d3ffee934e21054981769c322 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Wed, 27 Apr 2016 23:36:44 +0530 Subject: [PATCH 26/62] Consul Changes --- dockyard/common/membership/consul_driver.py | 35 ++++++++++++++------- dockyard/common/utils.py | 3 ++ test/consul_client | 1 + test/consul_server | 1 + 4 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 test/consul_client create mode 100644 test/consul_server diff --git a/dockyard/common/membership/consul_driver.py b/dockyard/common/membership/consul_driver.py index 52ed480..bd4850c 100644 --- a/dockyard/common/membership/consul_driver.py +++ b/dockyard/common/membership/consul_driver.py @@ -18,25 +18,38 @@ class Consul(Membership): def __init__(self): -# print help(consul.Consul.Agent.Service.__init__) - self.service = consul.Consul.Agent.Service(consul.Consul.Agent()) - self.catalog = consul.Consul.Catalog() + self.consul = consul.Consul() def register(self): ip = CONF['default']['host'] if ip == '0.0.0.0': ip = None - self.service.register(name=CONF.consul.service_name, - address = ip, - port = CONF.default.port, - tags = CONF.default.agent) - + self.consul.agent.service.register(name=CONF.consul.service_name, + address=ip, + port = CONF.default.port, + tags = ['master']) + + def _make_dict(self, service): + print service + ser = dict() + info = { + 'address': service['ServiceAddress'], + 'port': service['ServicePort'] + } + ser[service['Node']] = info + return ser; + def _get_services(self, services): - return services[1] + services_info = [] + + for service in services[1]: + services_info.append(self._make_dict(service)) + + return services_info - def get_all_host(self, tag='agent'): + def get_all_hosts(self, tag='agent'): """Returns all the members current agent sees. """ - services = self.catalog.service(CONF.consul.service_name, tag=tag) + services = self.consul.catalog.service('dockyard') return self._get_services(services) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index bdfdd2f..560349f 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -8,6 +8,9 @@ cfg.StrOpt('scheduler', default='round_robin.RoundRobinScheduler', help='Scheduler for the dockyard.'), + cfg.StrOpt('agent', + default='master', + help='Tags to be used.') ] CONF = cfg.CONF diff --git a/test/consul_client b/test/consul_client new file mode 100644 index 0000000..fcbf24c --- /dev/null +++ b/test/consul_client @@ -0,0 +1 @@ +consul agent -data-dir /tmp/consul -node=agent-two -bind=$1 -config-dir /etc/consul.d diff --git a/test/consul_server b/test/consul_server new file mode 100644 index 0000000..e91e422 --- /dev/null +++ b/test/consul_server @@ -0,0 +1 @@ +consul agent -server -bootstrap-expect 1 -data-dir /tmp/consul -node=agent-one -bind=$1 -config-dir /etc/consul.d From a8bdcb8360ac306e022072af11bb9455e63b3ae2 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Thu, 28 Apr 2016 21:00:41 +0530 Subject: [PATCH 27/62] Consul driver added --- .../common/container/scheduler/round_robin.py | 6 +- dockyard/common/membership/consul/__init__.py | 32 +++++++ .../common/membership/consul/consul_driver.py | 87 +++++++++++++++++++ dockyard/common/membership/consul_driver.py | 55 ------------ dockyard/common/utils.py | 9 +- etc/dockyard/dockyard.conf | 12 +++ 6 files changed, 138 insertions(+), 63 deletions(-) create mode 100644 dockyard/common/membership/consul/__init__.py create mode 100644 dockyard/common/membership/consul/consul_driver.py delete mode 100644 dockyard/common/membership/consul_driver.py diff --git a/dockyard/common/container/scheduler/round_robin.py b/dockyard/common/container/scheduler/round_robin.py index 89f9914..99a895e 100644 --- a/dockyard/common/container/scheduler/round_robin.py +++ b/dockyard/common/container/scheduler/round_robin.py @@ -1,6 +1,5 @@ from oslo_config import cfg -from dockyard.common.cluster import cluster from dockyard.common.container.scheduler.base import Scheduler @@ -8,10 +7,9 @@ class RoundRobinScheduler(Scheduler): count = -1 def __init__(self): - self.cluster = cluster.Cluster() + pass - def get_host(self): - hosts = self.cluster.get_hosts() + def get_host(self, hosts): num_hosts = len(hosts) self.count = (self.count + 1) % num_hosts return (hosts[self.count]) diff --git a/dockyard/common/membership/consul/__init__.py b/dockyard/common/membership/consul/__init__.py new file mode 100644 index 0000000..041b5b8 --- /dev/null +++ b/dockyard/common/membership/consul/__init__.py @@ -0,0 +1,32 @@ +from oslo_config import cfg + +CONSUL_SERVICE_OPT = [ + cfg.StrOpt('service_name', + default='dockyard', + help='Name of this service used by the consul') +] + +CONF = cfg.CONF +opt_group = cfg.OptGroup(name='consul', + title='Group for the consul parameters') +CONF.register_group(opt_group) +CONF.register_opts(CONSUL_SERVICE_OPT, opt_group) + + +DOCKER_SERVICE_OPT = [ + cfg.IPOpt('docker_host', + default='0.0.0.0', + help='IP address to which docker is binded'), + cfg.IntOpt('docker_port', + default='2375', + help='PORT on which docker is listening'), + cfg.StrOpt('docker_name', + default='docker', + help='Name of the service under which docker is registered') +] + +CONF = cfg.CONF +opt_group = cfg.OptGroup(name='docker', + title='Group for the docker server parameters') +CONF.register_group(opt_group) +CONF.register_opts(DOCKER_SERVICE_OPT, opt_group) diff --git a/dockyard/common/membership/consul/consul_driver.py b/dockyard/common/membership/consul/consul_driver.py new file mode 100644 index 0000000..6e1a20c --- /dev/null +++ b/dockyard/common/membership/consul/consul_driver.py @@ -0,0 +1,87 @@ +import consul +from oslo_config import cfg + +from dockyard.common.membership.base import Membership + +CONF = cfg.CONF + + +class Consul(Membership): + def __init__(self): + self.consul = consul.Consul() + + def _register_service(self, name, host, port, tags=None): + if not name or not host or not port: + # raise InsufficientInfo + pass + + self.consul.agent.service.register(name=name, + address=host, + port = port, + tags = tags) + + def _register_dockyard(self): + ip = CONF['default']['host'] + if ip == '0.0.0.0': + ip = None + + port = CONF['default']['port'] + tags = ['master'] + name = CONF['consul']['service_name'] + + self._register_service(name, ip, port, tags) + + def _register_docker(self): + ip = CONF['docker']['docker_host'] + if ip == '0.0.0.0': + ip = None + + port = CONF['docker']['docker_port'] + name = CONF['docker']['docker_name'] + + self._register_service(name, ip, port) + + + def register(self): + """ This method registers dockyard service information and + docker server running on this machine. + """ + self._register_dockyard() + self._register_docker() + + + + def _make_dict(self, service): + """ This method takes all the information returned by the consul + and returns ip address and port of the service. + """ + + if service['ServiceAddress']: + host = service['ServiceAddress'] + else: + host = service['Address'] + + + info = { + 'host': host, + 'port': service['ServicePort'] + } + + return info; + + def _get_services(self, services): + """ This method returns all the registered services. + """ + + services_info = [] + + for service in services[1]: + services_info.append(self._make_dict(service)) + + return services_info + + def get_all_hosts(self, tag='agent'): + """Returns all the members current agent sees. + """ + services = self.consul.catalog.service('docker') + return self._get_services(services) diff --git a/dockyard/common/membership/consul_driver.py b/dockyard/common/membership/consul_driver.py deleted file mode 100644 index bd4850c..0000000 --- a/dockyard/common/membership/consul_driver.py +++ /dev/null @@ -1,55 +0,0 @@ -import consul -from oslo_config import cfg - -from dockyard.common.membership.base import Membership - - -CONSUL_SERVICE_OPT = [ - cfg.StrOpt('service_name', - default='dockyard', - help='Name of this service used by the consul'), -] - -CONF = cfg.CONF -opt_group = cfg.OptGroup(name='consul', - title='Group for the consul parameters') -CONF.register_group(opt_group) -CONF.register_opts(CONSUL_SERVICE_OPT, opt_group) - -class Consul(Membership): - def __init__(self): - self.consul = consul.Consul() - - def register(self): - ip = CONF['default']['host'] - if ip == '0.0.0.0': - ip = None - - self.consul.agent.service.register(name=CONF.consul.service_name, - address=ip, - port = CONF.default.port, - tags = ['master']) - - def _make_dict(self, service): - print service - ser = dict() - info = { - 'address': service['ServiceAddress'], - 'port': service['ServicePort'] - } - ser[service['Node']] = info - return ser; - - def _get_services(self, services): - services_info = [] - - for service in services[1]: - services_info.append(self._make_dict(service)) - - return services_info - - def get_all_hosts(self, tag='agent'): - """Returns all the members current agent sees. - """ - services = self.consul.catalog.service('dockyard') - return self._get_services(services) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index 560349f..0376e1d 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -28,7 +28,7 @@ MEMBERSHIP_OPT = [ cfg.StrOpt('membership', - default='consul_driver.Consul', + default='consul.consul_driver.Consul', help='Scheduler for the dockyard.'), ] @@ -36,6 +36,7 @@ # Fetch scheduler defined in the configuration file and load it. membership_info = CONF.default.membership + # May be this path can be specified in the configuration file. membership_loc = 'dockyard.common.membership' membership_info = (('%s.%s') % (membership_loc, membership_info)) @@ -51,13 +52,13 @@ def get_config(group, option): def get_host(): -# print membership.get_all_hosts() - return scheduler.get_host() + hosts = membership.get_all_hosts() + return scheduler.get_host(hosts=hosts) def get_link(url, protocol='http'): host = get_host() - return link.make_url(host=host, protocol=protocol, url=url) + return link.make_url(host=host['host'], port=host['port'], url=url) def dispatch_get_request(url, protocol='http', query_params=None): diff --git a/etc/dockyard/dockyard.conf b/etc/dockyard/dockyard.conf index 4b8c9b5..d8caf86 100644 --- a/etc/dockyard/dockyard.conf +++ b/etc/dockyard/dockyard.conf @@ -21,3 +21,15 @@ agent = master [consul] service_name = 'dockyard' + +[docker] +# IP address to which docker container is binded. +docker_host = 0.0.0.0 + +# Port number on which docker container is listening. +docker_port = 2375 + +# Docker service name, it is given here because may be consul is used +# by other application also therefor dockyard can register under different +# service name +docker_name = 'docker' From b2f44255cad70d958e4c850c4cb6ac47394afd8f Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Thu, 28 Apr 2016 22:44:24 +0530 Subject: [PATCH 28/62] Added Consul exceptions --- dockyard/common/exception.py | 18 +++++++++++++++ .../common/membership/consul/consul_driver.py | 23 +++++++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 dockyard/common/exception.py diff --git a/dockyard/common/exception.py b/dockyard/common/exception.py new file mode 100644 index 0000000..e567321 --- /dev/null +++ b/dockyard/common/exception.py @@ -0,0 +1,18 @@ +class DockyardException(Exception): + def __init__(self, message, **kwargs): + if message: + self.message = message + + try: + self.message = self.message % message + except Exception as e: + pass + + super(DockyardException, self).__init__(self.message) + + +class IncompleteInfo(DockyardException): + message = ("%s is missing. Incomplete information.") + +class NoValidHostFound(DockyardException): + message = ("No Valid host for %s is found") diff --git a/dockyard/common/membership/consul/consul_driver.py b/dockyard/common/membership/consul/consul_driver.py index 6e1a20c..e4071b1 100644 --- a/dockyard/common/membership/consul/consul_driver.py +++ b/dockyard/common/membership/consul/consul_driver.py @@ -2,6 +2,7 @@ from oslo_config import cfg from dockyard.common.membership.base import Membership +from dockyard.common import exception CONF = cfg.CONF @@ -11,9 +12,18 @@ def __init__(self): self.consul = consul.Consul() def _register_service(self, name, host, port, tags=None): - if not name or not host or not port: - # raise InsufficientInfo - pass + if not name: + message = ('Service name to use for registering') + raise exception.IncompleteInfo(message) + +# if not host: +# message = ('IP address for the service to be used') +# raise exception.IncompleteInfo(message) + + if not port: + message = ('Port number used by the services to listen') + raise exception.Incompleteinfo(message) + self.consul.agent.service.register(name=name, address=host, @@ -22,6 +32,7 @@ def _register_service(self, name, host, port, tags=None): def _register_dockyard(self): ip = CONF['default']['host'] + if ip == '0.0.0.0': ip = None @@ -49,8 +60,6 @@ def register(self): self._register_dockyard() self._register_docker() - - def _make_dict(self, service): """ This method takes all the information returned by the consul and returns ip address and port of the service. @@ -84,4 +93,8 @@ def get_all_hosts(self, tag='agent'): """Returns all the members current agent sees. """ services = self.consul.catalog.service('docker') + if not services: + message = "No services are registered to the consul" + raise exception.NoValidHostFound(message) + return self._get_services(services) From 562efac1dfae2d38a4f464142fcbf356220870d2 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sat, 7 May 2016 21:24:56 -0400 Subject: [PATCH 29/62] Added dockyard-engine for processing --- dockyard/dockyard-engine/__init__.py | 0 dockyard/dockyard-engine/network/__init__.py | 0 .../network/drivers/__init__.py | 0 .../network/drivers/bridges/__init__.py | 0 .../network/drivers/bridges/base.py | 142 ++++++++++++++++++ .../network/drivers/bridges/test.py | 5 + 6 files changed, 147 insertions(+) create mode 100644 dockyard/dockyard-engine/__init__.py create mode 100644 dockyard/dockyard-engine/network/__init__.py create mode 100644 dockyard/dockyard-engine/network/drivers/__init__.py create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/__init__.py create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/base.py create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/test.py diff --git a/dockyard/dockyard-engine/__init__.py b/dockyard/dockyard-engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/dockyard-engine/network/__init__.py b/dockyard/dockyard-engine/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/dockyard-engine/network/drivers/__init__.py b/dockyard/dockyard-engine/network/drivers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/dockyard-engine/network/drivers/bridges/__init__.py b/dockyard/dockyard-engine/network/drivers/bridges/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/dockyard-engine/network/drivers/bridges/base.py b/dockyard/dockyard-engine/network/drivers/bridges/base.py new file mode 100644 index 0000000..f5d38aa --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/base.py @@ -0,0 +1,142 @@ +# This module does the common tasks required by driver, which are based +# on the bridges concepts. It does following tasks +# 1) Create Virtual Interface +# 2) Moving Virtual Interface into docker namespace +# 3) Assign IP address to the interface +# 4) Brings up network interface + +from pyroute2 import IPRoute + +class InsufficientInfo(Exception): + pass + +class InterfaceNotFound(Exception): + pass + +class FailedToMoveInterface(Exception): + pass + +class UnableToAssignIP(Exception): + pass + +class UnableToChangeState(Exception): + pass + +class InterfaceManager(object): + def __init__(self, id_, bridge=None): + self.ipr = IPRoute() + self.id = id_ + + if bridge: + self.bridge = bridge + + def _get_if_name(self, loc, kind='veth'): + """This method makes name of the interface in the specified format + for our application. + """ + + if not loc: + msg = "Location of the interface is not defined" + raise InsufficientInfo(msg) + + if loc == 'external': + loc = 'x' + elif loc == 'internal': + loc = 'i' + + return ('%s-%s-%s' % (kind, loc, self.id)); + + def create_link_pair(self, kind='veth', peer=None): + """Creates links, one for namespace and other out of namespace. + + ifname: Name of the interface. In this application, it will + contain id of the container, which will be appended + to the '[kind]-[i/x]' therefor it produce unique interface + name. + kind: It is the veth in our application. + peer: Name of the peer link, In this application link, out + of the namespace contains '[kind]-x-[id]' for external + link and '[kind]-i-[id]' for the namespace link. + """ + + self.ext_if = self._get_if_name(loc='external', kind=kind) + + if not peer: + self.int_if = self._get_if_name(loc='internal', kind=kind) + + try: + self.ipr.link('add', ifname=self.ext_if, + kind=kind, + peer=self.int_if) + except: + # A proper exception must be thrown here to inform kind of problem + # encountered + return False + + self.int_idx = self.ipr._get_index(ifname=self.int_if) + + return True + + def move_if(self, net_ns_fd): + """Moves interface to the namspace. + """ + try: + self.ipr.link('set', index=self.int_idx, + net_ns_fd=net_ns_fd) + except: + raise FailedToMoveInterface() + + def addr(self, address, mask, broadcast, net_ns_fd): + """Assign ip address + address: IPv4 or IPv6 address + mask: address mask + broadcast: Broadcast address + """ + try: + self.ipr.addr('add', self.int_idx, + address=address, + netmask=mask, + broadcast=broadcast, + net_ns_fd=net_ns_fd) + except: + raise UnableToAssignIP() + + def _change_state(self, command, state): + """Change state of the link or attach links. + """ + try: + self.ipr.link(command, index=self.int_idx, + state=state) + except: + raise UnableToChangeState() + + def _get_index(self, ifname): + """Get index of the bridge. + """ + try: + index = self.ipr.link_lookup(ifname=ifname)[0] + except: + raise InterfaceNotFound() + + return index + + def up(self, state): + """Brings interface ups. + """ + self._change_state(command="set", state="up") + + def down(self, state): + """Brings interface down. + """ + self._change_state(command="set", state="down") + + def attach_if(self, master=None): + """Attach interface to bridges. + """ + if not master: + self.master_int_idx = self._get_index(self.bridge) + else: + self.master_int_idx = self._get_index(master) + + self.ipr.link('set', index=self.int_idx, + master=self.master_int_idx) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test.py b/dockyard/dockyard-engine/network/drivers/bridges/test.py new file mode 100644 index 0000000..c4a2f4b --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/test.py @@ -0,0 +1,5 @@ +from base import InterfaceManager + +obj = InterfaceManager('1111', 'br-eth0') +obj.create_link_pair() +obj.attach_if() From cd1e5701bfb1d0a4e03b8a51e35e812707bc9925 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Fri, 13 May 2016 23:32:51 +0530 Subject: [PATCH 30/62] Changes in Linux bridge drivers --- .../network/drivers/bridges/base.py | 194 ++++++++++-------- .../network/drivers/bridges/linux.py | 75 +++++++ .../network/drivers/bridges/test.py | 12 +- requirement.txt | 2 + 4 files changed, 193 insertions(+), 90 deletions(-) create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/linux.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/base.py b/dockyard/dockyard-engine/network/drivers/bridges/base.py index f5d38aa..87493fa 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/base.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/base.py @@ -5,83 +5,120 @@ # 3) Assign IP address to the interface # 4) Brings up network interface +import abc from pyroute2 import IPRoute +from nsenter import Namespace -class InsufficientInfo(Exception): + +class BridgeDriversExceptions(Exception): pass -class InterfaceNotFound(Exception): + +class InterfaceNotFound(BridgeDriversExceptions): pass -class FailedToMoveInterface(Exception): + +class FailedToMoveInterface(BridgeDriversExceptions): pass -class UnableToAssignIP(Exception): + +class UnableToAssignIP(BridgeDriversExceptions): pass -class UnableToChangeState(Exception): + +class UnableToChangeState(BridgeDriversExceptions): pass -class InterfaceManager(object): - def __init__(self, id_, bridge=None): - self.ipr = IPRoute() - self.id = id_ - if bridge: - self.bridge = bridge +class InvalidState(BridgeDriversExceptions): + pass - def _get_if_name(self, loc, kind='veth'): - """This method makes name of the interface in the specified format - for our application. - """ - if not loc: - msg = "Location of the interface is not defined" - raise InsufficientInfo(msg) - - if loc == 'external': - loc = 'x' - elif loc == 'internal': - loc = 'i' - - return ('%s-%s-%s' % (kind, loc, self.id)); - - def create_link_pair(self, kind='veth', peer=None): - """Creates links, one for namespace and other out of namespace. - - ifname: Name of the interface. In this application, it will - contain id of the container, which will be appended - to the '[kind]-[i/x]' therefor it produce unique interface - name. - kind: It is the veth in our application. - peer: Name of the peer link, In this application link, out - of the namespace contains '[kind]-x-[id]' for external - link and '[kind]-i-[id]' for the namespace link. - """ +class BridgeManager(object): + def __init__(self): + pass - self.ext_if = self._get_if_name(loc='external', kind=kind) + @abc.abstractmethod + def create_link_pair(self, kind, peer=None): + pass - if not peer: - self.int_if = self._get_if_name(loc='internal', kind=kind) + @abc.abstractmethod + def attach_if(self, master, bridge): + pass - try: - self.ipr.link('add', ifname=self.ext_if, - kind=kind, - peer=self.int_if) - except: - # A proper exception must be thrown here to inform kind of problem - # encountered - return False + @abc.abstractmethod + def addr(self, address, mask, broadcast, net_ns_fd): + pass + + @abc.abstractmethod + def move_to_namespace(self, if_name, net_ns_fd): + pass + + @abc.abstractmethod + def change_state(self, if_name, state): + pass + + @abc.abstractmethod + def get_index(self, if_name): + pass + + +class Addr(object): + def __init__(self): + self.ipr = IPRoute() + + def add(self, command, index=None, address=None, mask=None, **kwargs): + self.ipr.addr(command, index=index, + address=address, + mask=mask, **kwargs) - self.int_idx = self.ipr._get_index(ifname=self.int_if) - return True +class Link(object): + __states = ['up', 'down'] - def move_if(self, net_ns_fd): + def __init__(self): + self.ipr = IPRoute() + + def add(self, command, ifname, peer, kind='veth'): + """Create link. + """ + self.ipr.link(command, + ifname=ifname, + kind=kind, + peer=peer) + + def set(self, command, + index=None, net_ns_fd=None, + state=None, master=None): + """Handles links. + """ + with Namespace(net_ns_fd, 'net'): + if state: + self.ipr.link(command, + index=index, + state=state) + else: + self.ipr.link(command, + index=index, + master=master) + + def lookup(self, ifname): + """Look up all the interfaces. + """ + return self.ipr.link_lookup(ifname=ifname) + + +class InterfaceManager(object): + def __init__(self): + self.link = Link() + self.addr = Addr() + + def move_to_namespace(self, if_name, net_ns_fd): """Moves interface to the namspace. """ + idx = self._get_index(if_name) try: - self.ipr.link('set', index=self.int_idx, + self.link.set('set', index=idx, net_ns_fd=net_ns_fd) except: raise FailedToMoveInterface() @@ -93,50 +130,35 @@ def addr(self, address, mask, broadcast, net_ns_fd): broadcast: Broadcast address """ try: - self.ipr.addr('add', self.int_idx, - address=address, - netmask=mask, - broadcast=broadcast, - net_ns_fd=net_ns_fd) + self.addr.add('add', self.int_idx, + address=address, + netmask=mask, + broadcast=broadcast, + net_ns_fd=net_ns_fd) except: raise UnableToAssignIP() - def _change_state(self, command, state): - """Change state of the link or attach links. - """ - try: - self.ipr.link(command, index=self.int_idx, - state=state) - except: - raise UnableToChangeState() - - def _get_index(self, ifname): + def get_index(self, if_name): """Get index of the bridge. """ try: - index = self.ipr.link_lookup(ifname=ifname)[0] + index = self.link.lookup(ifname=if_name)[0] except: raise InterfaceNotFound() - + return index - def up(self, state): + def change_state(self, if_name, state='up', net_ns_fd=None): """Brings interface ups. """ - self._change_state(command="set", state="up") - def down(self, state): - """Brings interface down. - """ - self._change_state(command="set", state="down") + if state not in self.__states: + msg = ("States has to be among %s" % (self.__states)) + raise InvalidState(msg) - def attach_if(self, master=None): - """Attach interface to bridges. - """ - if not master: - self.master_int_idx = self._get_index(self.bridge) - else: - self.master_int_idx = self._get_index(master) - - self.ipr.link('set', index=self.int_idx, - master=self.master_int_idx) + idx = self.get_index(if_name) + try: + self.link.set(command="set", state=state, index=idx) + except: + raise UnableToChangeState() + diff --git a/dockyard/dockyard-engine/network/drivers/bridges/linux.py b/dockyard/dockyard-engine/network/drivers/bridges/linux.py new file mode 100644 index 0000000..cfb6793 --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/linux.py @@ -0,0 +1,75 @@ +from pyroute2 import IPRoute +from base import InterfaceManager + + +class LinuxBridgeManager(object): + def __init__(self): + self.ln = IPRoute() + self.if_manager = InterfaceManager() + + def _get_if_name(self, id_, loc, kind='veth'): + """This method makes name of the interface in the specified format + for our application. + """ + + if not loc: + msg = "Location of the interface is not defined" + raise InsufficientInfo(msg) + + if loc == 'external': + loc = 'x' + elif loc == 'internal': + loc = 'i' + + return ('%s-%s-%s' % (kind, loc, id_)); + + + def create_link_pair(self, id_, kind='veth', peer=None): + """Creates links, one for namespace and other out of namespace. + + ifname: Name of the interface. In this application, it will + contain id of the container, which will be appended + to the '[kind]-[i/x]' therefor it produce unique interface + name. + kind: It is the veth in our application. + peer: Name of the peer link, In this application link, out + of the namespace contains '[kind]-x-[id]' for external + link and '[kind]-i-[id]' for the namespace link. + """ + + ext_if = self._get_if_name(id_, loc='external', kind=kind) + + if not peer: + int_if = self._get_if_name(id_, loc='internal', kind=kind) + + try: + self.ln.link('add', + ifname=ext_if, + kind=kind, + peer=int_if) + except Exception as e: + # A proper exception must be thrown here to inform kind of problem + # encountered + print e + return False + + return {'ext_if': ext_if, 'int_if': int_if} + + def attach_if(self, if_ext_idx, master_idx): + """Attach interface to bridges. + """ + # Define exception that unable to attach UnableToAttach + self.ln.link('set', index=if_ext_idx, + master=master_idx) + + def move_to_namespace(self, if_name, net_ns_fd): + self.if_manager.move_to_namespace(if_name, net_ns_fd) + + def addr(self, address, mask, broadcast, net_ns_fd): + self.if_manager.addr(address, mask, broadcast, net_ns_fd) + + def change_state(self, if_name, state='up', net_ns_fd=None): + self.if_manager.addr(if_name, state) + + def get_index(self, if_name): + self.if_manager.get_index(if_name) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test.py b/dockyard/dockyard-engine/network/drivers/bridges/test.py index c4a2f4b..ddfca0a 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/test.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/test.py @@ -1,5 +1,9 @@ -from base import InterfaceManager +from linux import LinuxBridgeManager -obj = InterfaceManager('1111', 'br-eth0') -obj.create_link_pair() -obj.attach_if() +obj = LinuxBridgeManager() +names = obj.create_link_pair(id_=5456) +print names +obj.attach_if(obj.get_index(names['ext_if']), obj.get_index('br100')) +obj.move_to_namespace(net_ns_fd=5456) +obj.change_state(names['int_if'], 5456) +obj.addr(address='192.168.100.1', mask='255.255.255.0', broadcast='192.168.100.255', net_ns_fd='5456') diff --git a/requirement.txt b/requirement.txt index 23e55f0..043919b 100644 --- a/requirement.txt +++ b/requirement.txt @@ -2,3 +2,5 @@ oslo.log oslo.config pecan python-consul +nsenter +pyroute2 From 6dac2c3c152a8b7da6accb05384138cbcc427866 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sat, 14 May 2016 00:04:06 +0530 Subject: [PATCH 31/62] corrections .. --- .../dockyard-engine/network/drivers/bridges/base.py | 12 ++++++------ .../dockyard-engine/network/drivers/bridges/linux.py | 8 ++++---- .../dockyard-engine/network/drivers/bridges/test.py | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/base.py b/dockyard/dockyard-engine/network/drivers/bridges/base.py index 87493fa..af43467 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/base.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/base.py @@ -74,7 +74,7 @@ def add(self, command, index=None, address=None, mask=None, **kwargs): class Link(object): - __states = ['up', 'down'] + allowed_states = ['up', 'down'] def __init__(self): self.ipr = IPRoute() @@ -113,10 +113,10 @@ def __init__(self): self.link = Link() self.addr = Addr() - def move_to_namespace(self, if_name, net_ns_fd): + def move_to_namespace(self, idx, net_ns_fd): """Moves interface to the namspace. """ - idx = self._get_index(if_name) + #idx = self.get_index(if_name) try: self.link.set('set', index=idx, net_ns_fd=net_ns_fd) @@ -151,9 +151,9 @@ def get_index(self, if_name): def change_state(self, if_name, state='up', net_ns_fd=None): """Brings interface ups. """ - - if state not in self.__states: - msg = ("States has to be among %s" % (self.__states)) + print state + if state not in self.link.allowed_states: + msg = ("States has to be among %s" % (self.link.allowed_states)) raise InvalidState(msg) idx = self.get_index(if_name) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/linux.py b/dockyard/dockyard-engine/network/drivers/bridges/linux.py index cfb6793..14c4552 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/linux.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/linux.py @@ -66,10 +66,10 @@ def move_to_namespace(self, if_name, net_ns_fd): self.if_manager.move_to_namespace(if_name, net_ns_fd) def addr(self, address, mask, broadcast, net_ns_fd): - self.if_manager.addr(address, mask, broadcast, net_ns_fd) + self.if_manager.addr.add(address, mask, broadcast, net_ns_fd) - def change_state(self, if_name, state='up', net_ns_fd=None): - self.if_manager.addr(if_name, state) + def change_state(self, idx, state='up', net_ns_fd=None): + self.if_manager.change_state(if_name=idx, state=state, net_ns_fd=net_ns_fd) def get_index(self, if_name): - self.if_manager.get_index(if_name) + return self.if_manager.get_index(if_name) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test.py b/dockyard/dockyard-engine/network/drivers/bridges/test.py index ddfca0a..a98e843 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/test.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/test.py @@ -1,9 +1,9 @@ from linux import LinuxBridgeManager obj = LinuxBridgeManager() -names = obj.create_link_pair(id_=5456) +names = obj.create_link_pair(id_=3400) print names obj.attach_if(obj.get_index(names['ext_if']), obj.get_index('br100')) -obj.move_to_namespace(net_ns_fd=5456) -obj.change_state(names['int_if'], 5456) -obj.addr(address='192.168.100.1', mask='255.255.255.0', broadcast='192.168.100.255', net_ns_fd='5456') +obj.move_to_namespace(obj.get_index(names['int_if']), net_ns_fd=3400) +obj.addr(address='192.168.100.1', mask='255.255.255.0', broadcast='192.168.100.255', net_ns_fd='3400') +obj.change_state(names['int_if'], 'up', 3400) From 819b7154719aa9860ba1740b284ede71eced191c Mon Sep 17 00:00:00 2001 From: root Date: Sat, 14 May 2016 18:35:29 +0530 Subject: [PATCH 32/62] Corrections --- .../network/drivers/bridges/base.py | 137 +++++++++++------- .../network/drivers/bridges/linux.py | 7 +- .../network/drivers/bridges/namespace.py | 72 +++++++++ .../bridges/network_driver_exceptions.py | 39 +++++ .../network/drivers/bridges/symlink.py | 19 +++ .../network/drivers/bridges/test.py | 13 +- .../network/drivers/bridges/test_ifs.py | 6 + .../network/drivers/bridges/test_namespace.py | 9 ++ 8 files changed, 239 insertions(+), 63 deletions(-) create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/namespace.py create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/symlink.py create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/test_ifs.py create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/test_namespace.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/base.py b/dockyard/dockyard-engine/network/drivers/bridges/base.py index af43467..ab94cbd 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/base.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/base.py @@ -1,37 +1,18 @@ # This module does the common tasks required by driver, which are based # on the bridges concepts. It does following tasks -# 1) Create Virtual Interface -# 2) Moving Virtual Interface into docker namespace -# 3) Assign IP address to the interface -# 4) Brings up network interface +# 1) Create Virtual Interface # 2) Moving Virtual Interface into docker namespace # 3) Assign IP address to the interface # 4) Brings up network interface import abc +from network_driver_exceptions import ( + AlreadyInNamespace, + FailedToMoveInterface, + InterfaceNotFound, + InvalidState, + NamespaceNotFound, + UnableToAssignIP, + UnableToChangeState) from pyroute2 import IPRoute -from nsenter import Namespace - - -class BridgeDriversExceptions(Exception): - pass - - -class InterfaceNotFound(BridgeDriversExceptions): - pass - - -class FailedToMoveInterface(BridgeDriversExceptions): - pass - - -class UnableToAssignIP(BridgeDriversExceptions): - pass - - -class UnableToChangeState(BridgeDriversExceptions): - pass - - -class InvalidState(BridgeDriversExceptions): - pass +from namespace import DockyardNamespace class BridgeManager(object): @@ -67,10 +48,11 @@ class Addr(object): def __init__(self): self.ipr = IPRoute() - def add(self, command, index=None, address=None, mask=None, **kwargs): + def add(self, command, index=None, address=None, mask=None, net_ns_fd=None): + print command, index, address, mask, net_ns_fd self.ipr.addr(command, index=index, - address=address, - mask=mask, **kwargs) + address=address, mask=mask, + net_ns_fd=net_ns_fd) class Link(object): @@ -87,20 +69,23 @@ def add(self, command, ifname, peer, kind='veth'): kind=kind, peer=peer) + # Use decorator to convert net_ns_fd to netns_name + # currently applying jugad def set(self, command, index=None, net_ns_fd=None, state=None, master=None): """Handles links. """ - with Namespace(net_ns_fd, 'net'): - if state: - self.ipr.link(command, - index=index, - state=state) - else: - self.ipr.link(command, - index=index, - master=master) + if state: + self.ipr.link(command, + index=index, + net_ns_fd = net_ns_fd, + state=state) + else: + self.ipr.link(command, + index=index, + net_ns_fd = net_ns_fd, + master=master) def lookup(self, ifname): """Look up all the interfaces. @@ -112,25 +97,70 @@ class InterfaceManager(object): def __init__(self): self.link = Link() self.addr = Addr() + self.netns = DockyardNamespace() + + def _does_if_exist(self, idx): + """This method checks whether a interface is in the + namespace. + + idx: index of the network interface + net_ns_fd: Namespace file descirptor. + + It returns True or False, depending on whether interface + is in namespace or not. + """ + + ifs = self.link.ipr.get_links() + ifs_idx = [x['index'] for x in ifs] + + if idx not in ifs_idx: + return True + else: + return False + + def _does_ns_exist(self, net_ns_fd): + return self.netns.does_exist(net_ns_fd) def move_to_namespace(self, idx, net_ns_fd): """Moves interface to the namspace. """ - #idx = self.get_index(if_name) - try: - self.link.set('set', index=idx, - net_ns_fd=net_ns_fd) - except: - raise FailedToMoveInterface() - def addr(self, address, mask, broadcast, net_ns_fd): + if self._does_if_exist(idx): + msg = ("%s interface does not exist" % (idx)) + raise InterfaceNotFound(msg) + + if not self._does_ns_exist(net_ns_fd): + try: + self.netns.attach_namespace(net_ns_fd) + except: + msg = ("%s Namespace does not exist" % (idx)) + raise NamespaceNotFound(msg) + + #try: + netns_name = self.netns.get_netns_name(net_ns_fd) + self.link.set('set', index=idx, + net_ns_fd=netns_name) + #except: + # raise FailedToMoveInterface() + + def addr(self, idx, address, mask, broadcast, net_ns_fd): """Assign ip address + idx: Device index address: IPv4 or IPv6 address mask: address mask broadcast: Broadcast address + net_ns_fd: process id for the container """ + + if not self._does_ns_exist(net_ns_fd): + try: + self.netns.attach_namespace(net_ns_fd) + except: + msg = ("%s Namespace does not exist" % (idx)) + raise NamespaceNotFound(msg) + try: - self.addr.add('add', self.int_idx, + self.addr.add('add', idx, address=address, netmask=mask, broadcast=broadcast, @@ -148,17 +178,16 @@ def get_index(self, if_name): return index - def change_state(self, if_name, state='up', net_ns_fd=None): + def change_state(self, if_idx, state='up', net_ns_fd=None): """Brings interface ups. """ - print state if state not in self.link.allowed_states: msg = ("States has to be among %s" % (self.link.allowed_states)) raise InvalidState(msg) - idx = self.get_index(if_name) try: - self.link.set(command="set", state=state, index=idx) + netns_name = self.netns.get_netns_name(net_ns_fd) + self.link.set(command="set", state=state, + index=if_idx, net_ns_fd=netns_name) except: raise UnableToChangeState() - diff --git a/dockyard/dockyard-engine/network/drivers/bridges/linux.py b/dockyard/dockyard-engine/network/drivers/bridges/linux.py index 14c4552..641c361 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/linux.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/linux.py @@ -1,5 +1,7 @@ from pyroute2 import IPRoute + from base import InterfaceManager +from network_driver_exceptions import InsufficientInfo class LinuxBridgeManager(object): @@ -51,7 +53,6 @@ def create_link_pair(self, id_, kind='veth', peer=None): # A proper exception must be thrown here to inform kind of problem # encountered print e - return False return {'ext_if': ext_if, 'int_if': int_if} @@ -65,8 +66,8 @@ def attach_if(self, if_ext_idx, master_idx): def move_to_namespace(self, if_name, net_ns_fd): self.if_manager.move_to_namespace(if_name, net_ns_fd) - def addr(self, address, mask, broadcast, net_ns_fd): - self.if_manager.addr.add(address, mask, broadcast, net_ns_fd) + def addr(self, idx, address, mask, broadcast, net_ns_fd): + self.if_manager.addr.add(idx, address, mask, broadcast, net_ns_fd) def change_state(self, idx, state='up', net_ns_fd=None): self.if_manager.change_state(if_name=idx, state=state, net_ns_fd=net_ns_fd) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/namespace.py b/dockyard/dockyard-engine/network/drivers/bridges/namespace.py new file mode 100644 index 0000000..1bb9c6d --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/namespace.py @@ -0,0 +1,72 @@ +# This module is responsible for making namespace descriptor to be +# present in /var/run/netns/. + +from network_driver_exceptions import UnableToAttachNamespace +from symlink import Symlink + +class Namespace(object): + def __init__(self): + pass + +class DockyardNamespace(Namespace): + _DOCKYARD_BASE_NETNS = "/var/run/dockyard/%d/ns/net" + _BASE_NETNS = "/var/run/netns" + + def __init__(self): + self.symlink = Symlink() + + def __attach_namespace(self, src, dst): + """This method attach to existing namespace. + src: It is the location of the file descriptor of the + namespace to attach. + dst: It is the location, where new file descriptor will be + created. + """ + #try: + self.symlink.create(src, dst) + #except: + # msg = ("Unable to attach to docker namespace") + # raise UnableToAttachNamespace(msg) + + def _get_src_netns(self, psid): + """This method return the orignial network namespace for + a docker container. + """ + path = ("/proc/%d/ns/net" % (psid)) + return path + + def _get_netns_loc(self, psid): + """Get network namespace location for the dockyard. + """ + return ("%s/dockyard_%d" % (self._BASE_NETNS, psid)) + + def attach_netns_namespace(self, psid): + """This method creates namespace for the pyroute library. + """ + src_netns = self._get_src_netns(psid) + dst_netns = ("%s/%s" % (self._BASE_NETNS, self.get_netns_name(psid))) + self.__attach_namespace(src_netns, dst_netns) + + def attach_dockyard_namespace(self, psid): + """This method creates namespace for the dockyard. + """ + src_netns = self._get_src_netns(psid) + dst_netns = self._get_netns_loc(psid) + self.__attach_namespace(src_netns, dst_netns) + + def attach_namespace(self, psid): + """Attach network namespace for dockyard. + """ + self.attach_netns_namespace(psid) + + def does_exist(self, psid): + """Checks whether network namespace exist or not. + """ + path = self._get_netns_loc(psid) + return self.symlink.is_symlink(path) + + def get_netns_name(self, psid): + """Returns name of the namespace. + """ + netns_name = ("dockyard_%d" % (psid)) + return netns_name diff --git a/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py b/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py new file mode 100644 index 0000000..0ff7c89 --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py @@ -0,0 +1,39 @@ +# This module holds all the exceptions used by bridge type driver. +class BridgeDriversExceptions(Exception): + pass + + +class NamespaceNotFound(BridgeDriversExceptions): + pass + + +class UnableToAttachNamespace(BridgeDriversExceptions): + pass + + +class InsufficientInfo(BridgeDriversExceptions): + pass + + +class AlreadyInNamespace(BridgeDriversExceptions): + pass + + +class InterfaceNotFound(BridgeDriversExceptions): + pass + + +class FailedToMoveInterface(BridgeDriversExceptions): + pass + + +class UnableToAssignIP(BridgeDriversExceptions): + pass + + +class UnableToChangeState(BridgeDriversExceptions): + pass + + +class InvalidState(BridgeDriversExceptions): + pass diff --git a/dockyard/dockyard-engine/network/drivers/bridges/symlink.py b/dockyard/dockyard-engine/network/drivers/bridges/symlink.py new file mode 100644 index 0000000..5226252 --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/symlink.py @@ -0,0 +1,19 @@ +# This module is responsible for creating links for the required +# namespace. + +import os + + +class Symlink(object): + def __init__(self): + pass + + def create(self, src, dst): + """This method is responsible for creating symbolic links. + """ + os.symlink(src, dst) + + def is_symlink(self, path): + """This method checks whether given path is symlink or not. + """ + return os.path.islink(path) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test.py b/dockyard/dockyard-engine/network/drivers/bridges/test.py index a98e843..cf11f66 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/test.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/test.py @@ -1,9 +1,10 @@ from linux import LinuxBridgeManager +from base import InterfaceManager obj = LinuxBridgeManager() -names = obj.create_link_pair(id_=3400) -print names -obj.attach_if(obj.get_index(names['ext_if']), obj.get_index('br100')) -obj.move_to_namespace(obj.get_index(names['int_if']), net_ns_fd=3400) -obj.addr(address='192.168.100.1', mask='255.255.255.0', broadcast='192.168.100.255', net_ns_fd='3400') -obj.change_state(names['int_if'], 'up', 3400) +#names = obj.create_link_pair(id_=29966) +#print names +#obj.attach_if(obj.get_index(names['ext_if']), obj.get_index('br100')) +#obj.move_to_namespace(obj.get_index(names['int_if']), net_ns_fd=29966) +obj.addr(62, address='192.168.100.1', mask='255.255.255.0', broadcast='192.168.100.255', net_ns_fd='29966') +obj.change_state(names['int_if'], 'up', 29966) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_ifs.py b/dockyard/dockyard-engine/network/drivers/bridges/test_ifs.py new file mode 100644 index 0000000..44777de --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/test_ifs.py @@ -0,0 +1,6 @@ +from pyroute2 import IPRoute +ip = IPRoute() +print ip.get_links()[0] +#print ip.get_links() +#for x in ip.get_links(): +# print x diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_namespace.py b/dockyard/dockyard-engine/network/drivers/bridges/test_namespace.py new file mode 100644 index 0000000..79e8b09 --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/test_namespace.py @@ -0,0 +1,9 @@ +from netns import Namespace +from pyroute2 import IPRoute + + +ip = IPRoute() +with Namespace(29966, 'net'): + ip.link.get_links() + +print ip From f317a7c6ce02fdf63dcee4b58f21c37981135508 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sun, 15 May 2016 00:03:49 +0530 Subject: [PATCH 33/62] Corrections in implementation --- .../network/drivers/bridges/base.py | 179 +++++++++++------- .../network/drivers/bridges/test.py | 21 +- 2 files changed, 120 insertions(+), 80 deletions(-) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/base.py b/dockyard/dockyard-engine/network/drivers/bridges/base.py index ab94cbd..8536bfb 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/base.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/base.py @@ -4,17 +4,18 @@ import abc from network_driver_exceptions import ( - AlreadyInNamespace, - FailedToMoveInterface, + AlreadyInNamespace, FailedToMoveInterface, InterfaceNotFound, InvalidState, NamespaceNotFound, UnableToAssignIP, UnableToChangeState) -from pyroute2 import IPRoute -from namespace import DockyardNamespace +from pyroute2 import IPDB, NetNS +from namespace import DockyardNamespace +# make sure this files interfaces are updated as per +# changes. class BridgeManager(object): def __init__(self): pass @@ -43,80 +44,122 @@ def change_state(self, if_name, state): def get_index(self, if_name): pass +class IPDBManager(object): + def __init__(self): + pass + + def open_ipdb(self, net_ns_fd=None): + self.ns = None + + if net_ns_fd: + self.ns = NetNS(net_ns_fd) + ipdb = IPDB(nl=self.ns) + else: + ipdb = IPDB() + + return ipdb + + + def close_ipdb(self, ipdb): + if self.ns: + self.ns.close() + + ipdb.commit() + ipdb.release() + class Addr(object): def __init__(self): - self.ipr = IPRoute() + self.ipdb_manager = IPDBManager() + + def add(self, ifname=None, address=None, + mask=None, net_ns_fd=None): + """Add ip address to the interface in namespace or outside + the name space. + """ + print ifname, address, mask, net_ns_fd + ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) - def add(self, command, index=None, address=None, mask=None, net_ns_fd=None): - print command, index, address, mask, net_ns_fd - self.ipr.addr(command, index=index, - address=address, mask=mask, - net_ns_fd=net_ns_fd) + if address: + address = ("%s/%d" % (address, mask)) + print address + with ipdb.interfaces[ifname] as interface: + if address: + interface.add_ip(address) + + self.ipdb_manager.close_ipdb(ipdb) + + def route(self): + pass class Link(object): allowed_states = ['up', 'down'] def __init__(self): - self.ipr = IPRoute() + self.ipdb_manager = IPDBManager() - def add(self, command, ifname, peer, kind='veth'): + def create(self, ifname, peer, kind='veth', net_ns_fd=None): """Create link. """ - self.ipr.link(command, - ifname=ifname, - kind=kind, - peer=peer) - - # Use decorator to convert net_ns_fd to netns_name - # currently applying jugad - def set(self, command, - index=None, net_ns_fd=None, - state=None, master=None): - """Handles links. + ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) + ipdb.create(ifname=ifname, kind=kind, peer=peer) + self.ipdb_manager.close_ipdb(ipdb) + + def move_to_namespace(self, ifname, net_ns_fd): + """Move an interface to the namespace. """ - if state: - self.ipr.link(command, - index=index, - net_ns_fd = net_ns_fd, - state=state) - else: - self.ipr.link(command, - index=index, - net_ns_fd = net_ns_fd, - master=master) + ipdb = self.ipdb_manager.open_ipdb() + with ipdb.interfaces[ifname] as interface: + interface.net_ns_fd = net_ns_fd - def lookup(self, ifname): - """Look up all the interfaces. - """ - return self.ipr.link_lookup(ifname=ifname) + self.ipdb_manager.close_ipdb(ipdb) + def set_state(self, ifname, net_ns_fd=None, state=None): + """Set state of the interface up/down in the namespace. + """ -class InterfaceManager(object): - def __init__(self): - self.link = Link() - self.addr = Addr() - self.netns = DockyardNamespace() + ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) + with ipdb.interfaces[ifname] as interface: + getattr(interface, state)() + + self.ipdb_manager.close_ipdb(ipdb) + + def get_ifs(self, net_ns_fd=None): + """Look up all the interfaces. + """ + ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) + ifs = [x for x in ipdb.interfaces if isinstance(x, str)] + self.ipdb_manager.close_ipdb(ipdb) + return ifs - def _does_if_exist(self, idx): + def does_if_exist(self, ifname, net_ns_fd=None): """This method checks whether a interface is in the namespace. - idx: index of the network interface + ifname: interface name of the network interface net_ns_fd: Namespace file descirptor. It returns True or False, depending on whether interface is in namespace or not. """ - ifs = self.link.ipr.get_links() - ifs_idx = [x['index'] for x in ifs] - - if idx not in ifs_idx: - return True - else: + ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) + try: + ipdb.interfaces[ifname] + except: return False + else: + self.ipdb_manager.close_ipdb(ipdb) + + return True + + +class InterfaceManager(object): + def __init__(self): + self.link = Link() + self.addr = Addr() + self.netns = DockyardNamespace() def _does_ns_exist(self, net_ns_fd): return self.netns.does_exist(net_ns_fd) @@ -136,14 +179,14 @@ def move_to_namespace(self, idx, net_ns_fd): msg = ("%s Namespace does not exist" % (idx)) raise NamespaceNotFound(msg) - #try: - netns_name = self.netns.get_netns_name(net_ns_fd) - self.link.set('set', index=idx, - net_ns_fd=netns_name) - #except: - # raise FailedToMoveInterface() - - def addr(self, idx, address, mask, broadcast, net_ns_fd): + try: + netns_name = self.netns.get_netns_name(net_ns_fd) + self.link.move_to_namespace(ifname=ifname, + net_ns_fd=netns_name) + except: + raise FailedToMoveInterface() + + def addr(self, ifname, address, mask, broadcast, net_ns_fd): """Assign ip address idx: Device index address: IPv4 or IPv6 address @@ -156,11 +199,11 @@ def addr(self, idx, address, mask, broadcast, net_ns_fd): try: self.netns.attach_namespace(net_ns_fd) except: - msg = ("%s Namespace does not exist" % (idx)) + msg = ("%s Namespace does not exist" % (net_ns_fd)) raise NamespaceNotFound(msg) try: - self.addr.add('add', idx, + self.addr.add(ifname=ifname, address=address, netmask=mask, broadcast=broadcast, @@ -168,17 +211,7 @@ def addr(self, idx, address, mask, broadcast, net_ns_fd): except: raise UnableToAssignIP() - def get_index(self, if_name): - """Get index of the bridge. - """ - try: - index = self.link.lookup(ifname=if_name)[0] - except: - raise InterfaceNotFound() - - return index - - def change_state(self, if_idx, state='up', net_ns_fd=None): + def change_state(self, if_name, state='up', net_ns_fd=None): """Brings interface ups. """ if state not in self.link.allowed_states: @@ -187,7 +220,7 @@ def change_state(self, if_idx, state='up', net_ns_fd=None): try: netns_name = self.netns.get_netns_name(net_ns_fd) - self.link.set(command="set", state=state, - index=if_idx, net_ns_fd=netns_name) + self.link.set_state(state=state, ifname=if_name, + net_ns_fd=netns_name) except: raise UnableToChangeState() diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test.py b/dockyard/dockyard-engine/network/drivers/bridges/test.py index cf11f66..c11344d 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/test.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/test.py @@ -1,10 +1,17 @@ from linux import LinuxBridgeManager -from base import InterfaceManager +from base import Link, Addr -obj = LinuxBridgeManager() -#names = obj.create_link_pair(id_=29966) -#print names +ifname='br1000' +peer = 'brx1000' +net_ns_fd='dockyard_6063' +obj = Link() +obj.create(ifname, peer) +#names = obj.create_link_pair(id_=6066) #obj.attach_if(obj.get_index(names['ext_if']), obj.get_index('br100')) -#obj.move_to_namespace(obj.get_index(names['int_if']), net_ns_fd=29966) -obj.addr(62, address='192.168.100.1', mask='255.255.255.0', broadcast='192.168.100.255', net_ns_fd='29966') -obj.change_state(names['int_if'], 'up', 29966) +print obj.get_ifs() +obj.move_to_namespace(ifname,net_ns_fd) +#for x in obj.lookup(net_ns_fd=net_ns_fd): +# print x +obj.set_state(ifname, net_ns_fd, state='up') +Addr().add(ifname, address='192.168.100.1', mask=24, net_ns_fd='dockyard_6063') +#obj.change_state(names['int_if'], 'up', 6066) From 2e5099c3a32531190d402164655afd1681bbd279 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Tue, 17 May 2016 22:18:37 +0530 Subject: [PATCH 34/62] Modifications --- .../network/drivers/bridges/base.py | 18 +++++++-------- .../network/drivers/bridges/linux.py | 23 ++++++++++--------- .../network/drivers/bridges/test.py | 16 ++++++------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/base.py b/dockyard/dockyard-engine/network/drivers/bridges/base.py index 8536bfb..98b1463 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/base.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/base.py @@ -77,12 +77,11 @@ def add(self, ifname=None, address=None, """Add ip address to the interface in namespace or outside the name space. """ - print ifname, address, mask, net_ns_fd ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) if address: address = ("%s/%d" % (address, mask)) - print address + with ipdb.interfaces[ifname] as interface: if address: interface.add_ip(address) @@ -164,29 +163,29 @@ def __init__(self): def _does_ns_exist(self, net_ns_fd): return self.netns.does_exist(net_ns_fd) - def move_to_namespace(self, idx, net_ns_fd): + def move_to_namespace(self, if_name, net_ns_fd): """Moves interface to the namspace. """ - if self._does_if_exist(idx): - msg = ("%s interface does not exist" % (idx)) + if self._does_if_exist(if_name): + msg = ("%s interface does not exist" % (if_name)) raise InterfaceNotFound(msg) if not self._does_ns_exist(net_ns_fd): try: self.netns.attach_namespace(net_ns_fd) except: - msg = ("%s Namespace does not exist" % (idx)) + msg = ("%s Namespace does not exist" % (if_name)) raise NamespaceNotFound(msg) try: netns_name = self.netns.get_netns_name(net_ns_fd) - self.link.move_to_namespace(ifname=ifname, + self.link.move_to_namespace(if_name=if_name, net_ns_fd=netns_name) except: raise FailedToMoveInterface() - def addr(self, ifname, address, mask, broadcast, net_ns_fd): + def addr(self, if_name, address, mask, broadcast, net_ns_fd): """Assign ip address idx: Device index address: IPv4 or IPv6 address @@ -203,7 +202,7 @@ def addr(self, ifname, address, mask, broadcast, net_ns_fd): raise NamespaceNotFound(msg) try: - self.addr.add(ifname=ifname, + self.addr.add(if_name=if_name, address=address, netmask=mask, broadcast=broadcast, @@ -224,3 +223,4 @@ def change_state(self, if_name, state='up', net_ns_fd=None): net_ns_fd=netns_name) except: raise UnableToChangeState() + diff --git a/dockyard/dockyard-engine/network/drivers/bridges/linux.py b/dockyard/dockyard-engine/network/drivers/bridges/linux.py index 641c361..a190ff8 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/linux.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/linux.py @@ -1,7 +1,9 @@ from pyroute2 import IPRoute from base import InterfaceManager -from network_driver_exceptions import InsufficientInfo +from network_driver_exceptions import ( + InsufficientInfo, + UnableToCreateInterface) class LinuxBridgeManager(object): @@ -49,28 +51,27 @@ def create_link_pair(self, id_, kind='veth', peer=None): ifname=ext_if, kind=kind, peer=int_if) - except Exception as e: - # A proper exception must be thrown here to inform kind of problem - # encountered - print e + except: + msg = ("Unable to create %s interface of %s kind" % (ifname, kind)) + raise UnableToCreateInterface(msg) return {'ext_if': ext_if, 'int_if': int_if} - def attach_if(self, if_ext_idx, master_idx): + def attach_if(self, if_ext_idx, master_if_name): """Attach interface to bridges. """ # Define exception that unable to attach UnableToAttach self.ln.link('set', index=if_ext_idx, - master=master_idx) + master=master_if_name) def move_to_namespace(self, if_name, net_ns_fd): self.if_manager.move_to_namespace(if_name, net_ns_fd) - def addr(self, idx, address, mask, broadcast, net_ns_fd): - self.if_manager.addr.add(idx, address, mask, broadcast, net_ns_fd) + def addr(self, if_name, address, mask, broadcast, net_ns_fd): + self.if_manager.addr.add(if_name, address, mask, broadcast, net_ns_fd) - def change_state(self, idx, state='up', net_ns_fd=None): - self.if_manager.change_state(if_name=idx, state=state, net_ns_fd=net_ns_fd) + def change_state(self, if_name, state='up', net_ns_fd=None): + self.if_manager.change_state(if_name=if_name, state=state, net_ns_fd=net_ns_fd) def get_index(self, if_name): return self.if_manager.get_index(if_name) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test.py b/dockyard/dockyard-engine/network/drivers/bridges/test.py index c11344d..595db61 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/test.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/test.py @@ -4,14 +4,14 @@ ifname='br1000' peer = 'brx1000' net_ns_fd='dockyard_6063' -obj = Link() -obj.create(ifname, peer) -#names = obj.create_link_pair(id_=6066) -#obj.attach_if(obj.get_index(names['ext_if']), obj.get_index('br100')) -print obj.get_ifs() -obj.move_to_namespace(ifname,net_ns_fd) +obj = LinuxBridgeManager() +#obj.create(ifname, peer) +names = obj.create_link_pair(id_=6066) +obj.attach_if(obj.get_index(names['ext_if']), 'br100') +#print obj.get_ifs() +obj.move_to_namespace(ifname['int_if'],net_ns_fd) #for x in obj.lookup(net_ns_fd=net_ns_fd): # print x -obj.set_state(ifname, net_ns_fd, state='up') +#obj.set_state(ifname, net_ns_fd, state='up') +obj.change_state(names['int_if'], 'up', 6066) Addr().add(ifname, address='192.168.100.1', mask=24, net_ns_fd='dockyard_6063') -#obj.change_state(names['int_if'], 'up', 6066) From d7268cbb901ac29119429dc39a7a35e027fd8be1 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Wed, 18 May 2016 21:48:55 +0530 Subject: [PATCH 35/62] Corrections in bridge driver --- .../network/drivers/bridges/base.py | 38 ++++++++++++++----- .../network/drivers/bridges/linux.py | 14 +++---- .../network/drivers/bridges/namespace.py | 10 ++--- .../bridges/network_driver_exceptions.py | 3 ++ .../network/drivers/bridges/test.py | 23 +++++------ 5 files changed, 56 insertions(+), 32 deletions(-) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/base.py b/dockyard/dockyard-engine/network/drivers/bridges/base.py index 98b1463..e93e236 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/base.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/base.py @@ -73,11 +73,11 @@ def __init__(self): self.ipdb_manager = IPDBManager() def add(self, ifname=None, address=None, - mask=None, net_ns_fd=None): + mask=None, broadcast=None, net_ns_fd=None): """Add ip address to the interface in namespace or outside the name space. """ - ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) + ipdb = self.ipdb_manager.open_ipdb(net_ns_fd=net_ns_fd) if address: address = ("%s/%d" % (address, mask)) @@ -88,9 +88,6 @@ def add(self, ifname=None, address=None, self.ipdb_manager.close_ipdb(ipdb) - def route(self): - pass - class Link(object): allowed_states = ['up', 'down'] @@ -119,6 +116,7 @@ def set_state(self, ifname, net_ns_fd=None, state=None): """ ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) + print ifname, net_ns_fd, state with ipdb.interfaces[ifname] as interface: getattr(interface, state)() @@ -154,6 +152,25 @@ def does_if_exist(self, ifname, net_ns_fd=None): return True +class IPManager(object): + def __init__(self): + """Manages IP manager. + """ + self.netns = DockyardNamespace() + self.addr = Addr() + + def assign_ip(self, ifname, address, mask, broadcast=None, net_ns_fd=None): + """Assign ip address. + :ifname: Assign ip address to this interface. + :address: Assign this ip address. + :mask: for the network + :broadcast: broadcast address for the network. + :net_ns_fd: network file descriptor or namespace. + """ + netns_name = self.netns.get_netns_name(psid=net_ns_fd) + self.addr.add(ifname, address, mask, broadcast, net_ns_fd=netns_name) + + class InterfaceManager(object): def __init__(self): self.link = Link() @@ -163,11 +180,14 @@ def __init__(self): def _does_ns_exist(self, net_ns_fd): return self.netns.does_exist(net_ns_fd) + def _does_if_exist(self, if_name): + return self.link.does_if_exist(if_name) + def move_to_namespace(self, if_name, net_ns_fd): """Moves interface to the namspace. """ - if self._does_if_exist(if_name): + if not self._does_if_exist(if_name): msg = ("%s interface does not exist" % (if_name)) raise InterfaceNotFound(msg) @@ -179,8 +199,8 @@ def move_to_namespace(self, if_name, net_ns_fd): raise NamespaceNotFound(msg) try: - netns_name = self.netns.get_netns_name(net_ns_fd) - self.link.move_to_namespace(if_name=if_name, + netns_name = self.netns.get_netns_name(net_ns_fd) + self.link.move_to_namespace(ifname=if_name, net_ns_fd=netns_name) except: raise FailedToMoveInterface() @@ -218,7 +238,7 @@ def change_state(self, if_name, state='up', net_ns_fd=None): raise InvalidState(msg) try: - netns_name = self.netns.get_netns_name(net_ns_fd) + netns_name = self.netns.get_netns_name(psid=net_ns_fd) self.link.set_state(state=state, ifname=if_name, net_ns_fd=netns_name) except: diff --git a/dockyard/dockyard-engine/network/drivers/bridges/linux.py b/dockyard/dockyard-engine/network/drivers/bridges/linux.py index a190ff8..2107565 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/linux.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/linux.py @@ -1,6 +1,6 @@ from pyroute2 import IPRoute -from base import InterfaceManager +from base import InterfaceManager, IPManager from network_driver_exceptions import ( InsufficientInfo, UnableToCreateInterface) @@ -10,6 +10,7 @@ class LinuxBridgeManager(object): def __init__(self): self.ln = IPRoute() self.if_manager = InterfaceManager() + self.ip_manager = IPManager() def _get_if_name(self, id_, loc, kind='veth'): """This method makes name of the interface in the specified format @@ -52,7 +53,9 @@ def create_link_pair(self, id_, kind='veth', peer=None): kind=kind, peer=int_if) except: - msg = ("Unable to create %s interface of %s kind" % (ifname, kind)) + msg = ("Unable to create %s, %s interfaces of %s kind" % ( + ext_if, int_if, kind)) + raise UnableToCreateInterface(msg) return {'ext_if': ext_if, 'int_if': int_if} @@ -67,11 +70,8 @@ def attach_if(self, if_ext_idx, master_if_name): def move_to_namespace(self, if_name, net_ns_fd): self.if_manager.move_to_namespace(if_name, net_ns_fd) - def addr(self, if_name, address, mask, broadcast, net_ns_fd): - self.if_manager.addr.add(if_name, address, mask, broadcast, net_ns_fd) + def addr(self, if_name, address, mask, broadcast=None, net_ns_fd=None): + self.ip_manager.assign_ip(if_name, address, mask, broadcast, net_ns_fd) def change_state(self, if_name, state='up', net_ns_fd=None): self.if_manager.change_state(if_name=if_name, state=state, net_ns_fd=net_ns_fd) - - def get_index(self, if_name): - return self.if_manager.get_index(if_name) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/namespace.py b/dockyard/dockyard-engine/network/drivers/bridges/namespace.py index 1bb9c6d..aa21884 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/namespace.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/namespace.py @@ -22,11 +22,11 @@ def __attach_namespace(self, src, dst): dst: It is the location, where new file descriptor will be created. """ - #try: - self.symlink.create(src, dst) - #except: - # msg = ("Unable to attach to docker namespace") - # raise UnableToAttachNamespace(msg) + try: + self.symlink.create(src, dst) + except: + msg = ("Unable to attach to docker namespace") + raise UnableToAttachNamespace(msg) def _get_src_netns(self, psid): """This method return the orignial network namespace for diff --git a/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py b/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py index 0ff7c89..55544fe 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py @@ -37,3 +37,6 @@ class UnableToChangeState(BridgeDriversExceptions): class InvalidState(BridgeDriversExceptions): pass + +class UnableToCreateInterface(BridgeDriversExceptions): + pass diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test.py b/dockyard/dockyard-engine/network/drivers/bridges/test.py index 595db61..50f0708 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/test.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/test.py @@ -1,17 +1,18 @@ from linux import LinuxBridgeManager -from base import Link, Addr +from base import Link -ifname='br1000' -peer = 'brx1000' -net_ns_fd='dockyard_6063' -obj = LinuxBridgeManager() +#ifname='br1000' +#peer = 'brx1000' +#net_ns_fd='dockyard_4235' +#obj = LinuxBridgeManager() +#obj = Link() #obj.create(ifname, peer) -names = obj.create_link_pair(id_=6066) -obj.attach_if(obj.get_index(names['ext_if']), 'br100') +#names = obj.create_link_pair(id_=4235) +#obj.attach_if(names['ext_if'], 'br100') #print obj.get_ifs() -obj.move_to_namespace(ifname['int_if'],net_ns_fd) +#obj.move_to_namespace(names['int_if'],net_ns_fd) #for x in obj.lookup(net_ns_fd=net_ns_fd): # print x -#obj.set_state(ifname, net_ns_fd, state='up') -obj.change_state(names['int_if'], 'up', 6066) -Addr().add(ifname, address='192.168.100.1', mask=24, net_ns_fd='dockyard_6063') +#obj.set_state('veth-i-4235', net_ns_fd, state='up') +#obj.change_state(names['int_if'], 'up', 4235) +#obj.addr(names['int_if'], address='192.168.100.1', mask=24, net_ns_fd=4235) From 8faeee751f2e8b665d4ee45798d9a341a385038e Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sat, 21 May 2016 00:36:13 +0530 Subject: [PATCH 36/62] Added suport for adding routes for the new network --- bugs/known_bugs.txt | 3 + .../network/drivers/bridges/base.py | 197 +++++++++++++----- .../network/drivers/bridges/linux.py | 67 ++++-- .../bridges/network_driver_exceptions.py | 2 + .../network/drivers/bridges/test.py | 18 -- .../network/drivers/bridges/test_ifs.py | 6 - .../network/drivers/bridges/test_linux.py | 11 + .../network/drivers/bridges/test_namespace.py | 9 - 8 files changed, 215 insertions(+), 98 deletions(-) delete mode 100644 dockyard/dockyard-engine/network/drivers/bridges/test.py delete mode 100644 dockyard/dockyard-engine/network/drivers/bridges/test_ifs.py create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/test_linux.py delete mode 100644 dockyard/dockyard-engine/network/drivers/bridges/test_namespace.py diff --git a/bugs/known_bugs.txt b/bugs/known_bugs.txt index 6a7e934..8536e76 100755 --- a/bugs/known_bugs.txt +++ b/bugs/known_bugs.txt @@ -1,3 +1,6 @@ ================================================================= Add known bugs in this file. Select a format to add bugs. ================================================================= +1) Add feature of receiving IP address from the DHCP +2) Add attach interface function attaching interface to the bridge +3) Method for Communication among docker processes running among cluster hosts diff --git a/dockyard/dockyard-engine/network/drivers/bridges/base.py b/dockyard/dockyard-engine/network/drivers/bridges/base.py index e93e236..0b26dd1 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/base.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/base.py @@ -9,6 +9,7 @@ InvalidState, NamespaceNotFound, UnableToAssignIP, + UnableToAddRoutes, UnableToChangeState) from pyroute2 import IPDB, NetNS @@ -33,17 +34,18 @@ def addr(self, address, mask, broadcast, net_ns_fd): pass @abc.abstractmethod - def move_to_namespace(self, if_name, net_ns_fd): + def move_to_namespace(self, ifname, net_ns_fd): pass @abc.abstractmethod - def change_state(self, if_name, state): + def change_state(self, ifname, state): pass @abc.abstractmethod - def get_index(self, if_name): + def get_index(self, ifname): pass + class IPDBManager(object): def __init__(self): pass @@ -88,6 +90,31 @@ def add(self, ifname=None, address=None, self.ipdb_manager.close_ipdb(ipdb) + def routes(self, dst, gateway, oif, net_ns_fd=None, **kwargs): + """Add routes to the namespace. + :dst: destination for which routes are being added. + :gateway: Gateway to be set. + :net_ns_fd: Network namespace file descriptor. + :kwargs: In case of advanced networking, additional parameters + might be provided through this option. + """ + ipdb = self.ipdb_manager.open_ipdb(net_ns_fd=net_ns_fd) + t = {} + t['dst'] = dst + t['gateway'] = gateway + t['oif'] = oif + + specs = t.copy() + specs.update(kwargs) + + with ipdb.routes[dst] as route: + for key, value in specs.iteritems(): + if value == dst: + continue + route[key] = value + + self.ipdb_manager.close_ipdb(ipdb) + class Link(object): allowed_states = ['up', 'down'] @@ -106,6 +133,7 @@ def move_to_namespace(self, ifname, net_ns_fd): """Move an interface to the namespace. """ ipdb = self.ipdb_manager.open_ipdb() + with ipdb.interfaces[ifname] as interface: interface.net_ns_fd = net_ns_fd @@ -114,9 +142,8 @@ def move_to_namespace(self, ifname, net_ns_fd): def set_state(self, ifname, net_ns_fd=None, state=None): """Set state of the interface up/down in the namespace. """ - ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) - print ifname, net_ns_fd, state + with ipdb.interfaces[ifname] as interface: getattr(interface, state)() @@ -130,6 +157,14 @@ def get_ifs(self, net_ns_fd=None): self.ipdb_manager.close_ipdb(ipdb) return ifs + def get_if(self, name, net_ns_fd=None): + """Look up all the interfaces. + """ + ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) + if_info = ipdb.interfaces[name] + self.ipdb_manager.close_ipdb(ipdb) + return if_info + def does_if_exist(self, ifname, net_ns_fd=None): """This method checks whether a interface is in the namespace. @@ -158,89 +193,149 @@ def __init__(self): """ self.netns = DockyardNamespace() self.addr = Addr() + self.link = Link() + + def _does_ns_exist(self, psid): + return self.netns.does_exist(psid) + + def _check_and_attach(self, psid): + if not self._does_ns_exist(psid): + try: + self.netns.attach_namespace(psid) + except: + msg = ("%s Namespace does not exist" % (psid)) + raise NamespaceNotFound(msg) + + return psid - def assign_ip(self, ifname, address, mask, broadcast=None, net_ns_fd=None): + def assign_ip(self, ifname, address, mask, broadcast=None, psid=None): """Assign ip address. :ifname: Assign ip address to this interface. :address: Assign this ip address. :mask: for the network :broadcast: broadcast address for the network. :net_ns_fd: network file descriptor or namespace. + + :raises NamespaceNotFound, UnableToAssignIP + """ + netns_name = None + + if psid: + psid = self._check_and_attach(psid) + netns_name = self.netns.get_netns_name(psid=psid) + + try: + self.addr.add(ifname, address, mask, + broadcast, net_ns_fd=netns_name) + except: + raise UnableToAssignIP() + + def add_routes(self, dst, gateway, oif_name, psid=None, **kwargs): + """Add routes to the namespace. + :dst: destination for which routes are being added. + :gateway: Gateway to be set. + :net_ns_fd: Network namespace file descriptor. + :kwargs: In case of advanced networking, additional parameters + might be provided through this option. """ - netns_name = self.netns.get_netns_name(psid=net_ns_fd) - self.addr.add(ifname, address, mask, broadcast, net_ns_fd=netns_name) + netns_name = None + + if psid: + psid = self._check_and_attach(psid) + netns_name = self.netns.get_netns_name(psid=psid) + + if self.link.does_if_exist(oif_name, net_ns_fd=netns_name): + oif_idx = self.link.get_if(oif_name, net_ns_fd=netns_name) + oif_idx = oif_idx['index'] + else: + msg = ("%s interface for setting default route in %s namespace " + "is not found" % (oif_name, netns_name)) + + raise InterfaceNotFound(msg) + + try: + if netns_name: + self.addr.routes(oif=oif_idx, dst=dst, gateway=gateway, + net_ns_fd=netns_name, **kwargs) + else: + self.addr.routes(oif=oif_idx, dst=dst, + gateway=gateway, **kwargs) + except: + msg = ("Unable to add route dst=%s, gateway=%s, network=%s " + "%s " % (dst, gateway, netns_name, kwargs)) + raise UnableToAddRoutes(msg) class InterfaceManager(object): def __init__(self): self.link = Link() - self.addr = Addr() self.netns = DockyardNamespace() - def _does_ns_exist(self, net_ns_fd): - return self.netns.does_exist(net_ns_fd) + def _does_ns_exist(self, psid): + return self.netns.does_exist(psid) - def _does_if_exist(self, if_name): - return self.link.does_if_exist(if_name) + def _does_if_exist(self, ifname, psid=None): + """Checks whether interface exist or not in a namespace. + :ifname: Name of the interface. + :psid: process id of the container. + + :returns True or False based on whether ifname exist or not. + """ + ns = False + + if not psid: + ns = self.link.does_if_exist(ifname) + else: + netns_name = self.netns.get_netns_name(ifname, psid) + ns = self.link.does_if_exist(ifname, net_ns_fd=netns_name) + + return ns - def move_to_namespace(self, if_name, net_ns_fd): + def move_to_namespace(self, ifname, psid): """Moves interface to the namspace. + :ifname: Interface name. + :psid: Process id for the docker container. + + :raises NamespaceNotFound, FailedToMoveInterface """ - if not self._does_if_exist(if_name): - msg = ("%s interface does not exist" % (if_name)) + if not self._does_if_exist(ifname): + msg = ("%s interface does not exist" % (ifname)) raise InterfaceNotFound(msg) - if not self._does_ns_exist(net_ns_fd): + if not self._does_ns_exist(psid): try: - self.netns.attach_namespace(net_ns_fd) + self.netns.attach_namespace(psid) except: - msg = ("%s Namespace does not exist" % (if_name)) + msg = ("%s Namespace does not exist" % (ifname)) raise NamespaceNotFound(msg) try: - netns_name = self.netns.get_netns_name(net_ns_fd) - self.link.move_to_namespace(ifname=if_name, - net_ns_fd=netns_name) + netns_name = self.netns.get_netns_name(psid) + self.link.move_to_namespace(ifname=ifname, + net_ns_fd=netns_name) except: raise FailedToMoveInterface() - def addr(self, if_name, address, mask, broadcast, net_ns_fd): - """Assign ip address - idx: Device index - address: IPv4 or IPv6 address - mask: address mask - broadcast: Broadcast address - net_ns_fd: process id for the container - """ - - if not self._does_ns_exist(net_ns_fd): - try: - self.netns.attach_namespace(net_ns_fd) - except: - msg = ("%s Namespace does not exist" % (net_ns_fd)) - raise NamespaceNotFound(msg) - - try: - self.addr.add(if_name=if_name, - address=address, - netmask=mask, - broadcast=broadcast, - net_ns_fd=net_ns_fd) - except: - raise UnableToAssignIP() - - def change_state(self, if_name, state='up', net_ns_fd=None): + def change_state(self, ifname, state='up', psid=None): """Brings interface ups. + :ifname: Interface name + :state: Expected state of the interface. + :psid: Process id of the docker process. + + :raises InvalidState, UnableToChangeState """ if state not in self.link.allowed_states: msg = ("States has to be among %s" % (self.link.allowed_states)) raise InvalidState(msg) + if psid: + netns_name = self.netns.get_netns_name(psid=psid) + else: + netns_name = None + try: - netns_name = self.netns.get_netns_name(psid=net_ns_fd) - self.link.set_state(state=state, ifname=if_name, + self.link.set_state(state=state, ifname=ifname, net_ns_fd=netns_name) except: raise UnableToChangeState() - diff --git a/dockyard/dockyard-engine/network/drivers/bridges/linux.py b/dockyard/dockyard-engine/network/drivers/bridges/linux.py index 2107565..10fe6fd 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/linux.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/linux.py @@ -12,7 +12,7 @@ def __init__(self): self.if_manager = InterfaceManager() self.ip_manager = IPManager() - def _get_if_name(self, id_, loc, kind='veth'): + def _get_ifname(self, id_, loc, prefix='dockyard'): """This method makes name of the interface in the specified format for our application. """ @@ -26,7 +26,7 @@ def _get_if_name(self, id_, loc, kind='veth'): elif loc == 'internal': loc = 'i' - return ('%s-%s-%s' % (kind, loc, id_)); + return ('%s-%s-%s' % (prefix, loc, id_)); def create_link_pair(self, id_, kind='veth', peer=None): @@ -42,10 +42,10 @@ def create_link_pair(self, id_, kind='veth', peer=None): link and '[kind]-i-[id]' for the namespace link. """ - ext_if = self._get_if_name(id_, loc='external', kind=kind) + ext_if = self._get_ifname(id_, loc='external') if not peer: - int_if = self._get_if_name(id_, loc='internal', kind=kind) + int_if = self._get_ifname(id_, loc='internal') try: self.ln.link('add', @@ -60,18 +60,57 @@ def create_link_pair(self, id_, kind='veth', peer=None): return {'ext_if': ext_if, 'int_if': int_if} - def attach_if(self, if_ext_idx, master_if_name): + # Make This method working + def attach_if(self, ifname, br_name): """Attach interface to bridges. """ - # Define exception that unable to attach UnableToAttach - self.ln.link('set', index=if_ext_idx, - master=master_if_name) + self.ln.link('set', index=ifname, + master=br_name) + + def move_to_namespace(self, ifname, psid): + """move an interface to the docker process namespace. + :ifname: Interface name + :psid: Docker processs name. + + :raises InterfaceNotFound, NamespaceNotFound, FailedTOMoveInterface + """ + + self.if_manager.move_to_namespace(ifname, psid) - def move_to_namespace(self, if_name, net_ns_fd): - self.if_manager.move_to_namespace(if_name, net_ns_fd) + def addr(self, ifname, address, mask, broadcast=None, psid=None): + """Assign ip address. + :ifname: Assign ip address to this interface. + :address: Assign this ip address. + :mask: for the network + :broadcast: broadcast address for the network. + :net_ns_fd: network file descriptor or namespace. + + :raises NamespaceNotFound, UnableToAssignIP + """ + self.ip_manager.assign_ip(ifname, address, mask, broadcast, psid) - def addr(self, if_name, address, mask, broadcast=None, net_ns_fd=None): - self.ip_manager.assign_ip(if_name, address, mask, broadcast, net_ns_fd) + def change_state(self, ifname, state='up', psid=None): + """Change state of the interface. + :ifname: Interface name + :state: Expected state of the interface valid values + are up, down. + :psid: process id for the docker container. + + :raises InvalidState, UnableToChangeState + """ + self.if_manager.change_state(ifname=ifname, state=state, psid=psid) + + def add_routes(self, oif_name, dst='default', gateway='0.0.0.0', + psid=None, **kwargs): + """Add routes to the namespace. + :dst: destination for which routes are being added. + :gateway: Gateway to be set. + :net_ns_fd: Network namespace file descriptor. + :kwargs: In case of advanced networking, additional parameters + might be provided through this option. + """ + if psid: + self.if_manager.netns.get_netns_name(psid=psid) - def change_state(self, if_name, state='up', net_ns_fd=None): - self.if_manager.change_state(if_name=if_name, state=state, net_ns_fd=net_ns_fd) + self.ip_manager.add_routes(oif_name=oif_name, dst=dst, gateway=gateway, + psid=psid, **kwargs) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py b/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py index 55544fe..42a7d4b 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py @@ -6,6 +6,8 @@ class BridgeDriversExceptions(Exception): class NamespaceNotFound(BridgeDriversExceptions): pass +class UnableToAddRoutes(BridgeDriversExceptions): + pass class UnableToAttachNamespace(BridgeDriversExceptions): pass diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test.py b/dockyard/dockyard-engine/network/drivers/bridges/test.py deleted file mode 100644 index 50f0708..0000000 --- a/dockyard/dockyard-engine/network/drivers/bridges/test.py +++ /dev/null @@ -1,18 +0,0 @@ -from linux import LinuxBridgeManager -from base import Link - -#ifname='br1000' -#peer = 'brx1000' -#net_ns_fd='dockyard_4235' -#obj = LinuxBridgeManager() -#obj = Link() -#obj.create(ifname, peer) -#names = obj.create_link_pair(id_=4235) -#obj.attach_if(names['ext_if'], 'br100') -#print obj.get_ifs() -#obj.move_to_namespace(names['int_if'],net_ns_fd) -#for x in obj.lookup(net_ns_fd=net_ns_fd): -# print x -#obj.set_state('veth-i-4235', net_ns_fd, state='up') -#obj.change_state(names['int_if'], 'up', 4235) -#obj.addr(names['int_if'], address='192.168.100.1', mask=24, net_ns_fd=4235) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_ifs.py b/dockyard/dockyard-engine/network/drivers/bridges/test_ifs.py deleted file mode 100644 index 44777de..0000000 --- a/dockyard/dockyard-engine/network/drivers/bridges/test_ifs.py +++ /dev/null @@ -1,6 +0,0 @@ -from pyroute2 import IPRoute -ip = IPRoute() -print ip.get_links()[0] -#print ip.get_links() -#for x in ip.get_links(): -# print x diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_linux.py b/dockyard/dockyard-engine/network/drivers/bridges/test_linux.py new file mode 100644 index 0000000..c086f80 --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/test_linux.py @@ -0,0 +1,11 @@ +from linux import LinuxBridgeManager + +obj = LinuxBridgeManager() +psid=4067 +ifs = {'int_if': ('dockyard-i-%d' % psid), + 'ext_if': ('dockyard-x-%d' % psid)} +obj.create_link_pair(psid) +obj.move_to_namespace(ifname=ifs['int_if'], psid=psid) +obj.change_state(ifname=ifs['int_if'], state='up', psid=psid) +obj.addr(ifname=ifs['int_if'], address='192.168.100.4', mask=24, psid=psid) +obj.add_routes(dst='default', gateway='192.168.100.1', psid=psid, oif_name=ifs['int_if']) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_namespace.py b/dockyard/dockyard-engine/network/drivers/bridges/test_namespace.py deleted file mode 100644 index 79e8b09..0000000 --- a/dockyard/dockyard-engine/network/drivers/bridges/test_namespace.py +++ /dev/null @@ -1,9 +0,0 @@ -from netns import Namespace -from pyroute2 import IPRoute - - -ip = IPRoute() -with Namespace(29966, 'net'): - ip.link.get_links() - -print ip From 6efb915266dec7412e6da98d0c6ea36c52a6f637 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 21 May 2016 13:13:15 +0530 Subject: [PATCH 37/62] Fixed for issues 20, 24, 25, 27, 28, 29, 31 --- .../network/drivers/bridges/base.py | 53 +++++++++++----- .../network/drivers/bridges/linux.py | 61 ++++++++++--------- .../network/drivers/bridges/namespace.py | 6 ++ .../bridges/network_driver_exceptions.py | 4 +- .../network/drivers/bridges/symlink.py | 6 ++ .../network/drivers/bridges/test_add_port.py | 3 + .../network/drivers/bridges/test_cleanup.py | 6 ++ .../network/drivers/bridges/test_linux.py | 12 ++-- .../network/drivers/bridges/utils.py | 32 ++++++++++ 9 files changed, 134 insertions(+), 49 deletions(-) create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/test_add_port.py create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/test_cleanup.py create mode 100644 dockyard/dockyard-engine/network/drivers/bridges/utils.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/base.py b/dockyard/dockyard-engine/network/drivers/bridges/base.py index 0b26dd1..b9753de 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/base.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/base.py @@ -186,6 +186,18 @@ def does_if_exist(self, ifname, net_ns_fd=None): return True + def attach_port(self, ifname, bridge, net_ns_fd=None): + """This method attach interface to the bridge. + :ifname: interface name to attach to the bridge. + :bridge: Name of the bridge to which attach interface. + """ + ipdb = self.ipdb_manager.open_ipdb(net_ns_fd=net_ns_fd) + + with ipdb.interfaces[bridge] as br: + br.add_port(ipdb.interfaces[ifname]) + + self.ipdb_manager.close_ipdb(ipdb) + class IPManager(object): def __init__(self): @@ -202,8 +214,8 @@ def _check_and_attach(self, psid): if not self._does_ns_exist(psid): try: self.netns.attach_namespace(psid) - except: - msg = ("%s Namespace does not exist" % (psid)) + except Exception as e: + msg = ("%s Namespace does not exist. ERROR: %s" % (psid, e)) raise NamespaceNotFound(msg) return psid @@ -227,8 +239,11 @@ def assign_ip(self, ifname, address, mask, broadcast=None, psid=None): try: self.addr.add(ifname, address, mask, broadcast, net_ns_fd=netns_name) - except: - raise UnableToAssignIP() + except Exception as e: + msg = ("Unable to assign ip %s to %s interface in %s namespace." + "ERROR: %s" % (address, ifname, net_ns_fd, e)) + + raise UnableToAssignIP(msg) def add_routes(self, dst, gateway, oif_name, psid=None, **kwargs): """Add routes to the namespace. @@ -260,9 +275,11 @@ def add_routes(self, dst, gateway, oif_name, psid=None, **kwargs): else: self.addr.routes(oif=oif_idx, dst=dst, gateway=gateway, **kwargs) - except: - msg = ("Unable to add route dst=%s, gateway=%s, network=%s " - "%s " % (dst, gateway, netns_name, kwargs)) + except Exception as e: + msg = ("Unable to add gateway %s for destination %s in namespace " + "%s for interface %d. ERROR: %s" % (gateway, dst, + netns_name, oif_idx, e)) + raise UnableToAddRoutes(msg) @@ -306,16 +323,19 @@ def move_to_namespace(self, ifname, psid): if not self._does_ns_exist(psid): try: self.netns.attach_namespace(psid) - except: - msg = ("%s Namespace does not exist" % (ifname)) + except Exception as e: + msg = ("%s Namespace does not exist. ERROR: %s" % (ifname, e)) raise NamespaceNotFound(msg) try: netns_name = self.netns.get_netns_name(psid) self.link.move_to_namespace(ifname=ifname, net_ns_fd=netns_name) - except: - raise FailedToMoveInterface() + except Exception as e: + msg = ("Failed to move %s interface in %s namespace. ERROR: %s" % ( + ifname, netns_name, e)) + + raise FailedToMoveInterface(msg) def change_state(self, ifname, state='up', psid=None): """Brings interface ups. @@ -326,7 +346,9 @@ def change_state(self, ifname, state='up', psid=None): :raises InvalidState, UnableToChangeState """ if state not in self.link.allowed_states: - msg = ("States has to be among %s" % (self.link.allowed_states)) + msg = ("States has to be among %s but Received %s state" % ( + self.link.allowed_states, state)) + raise InvalidState(msg) if psid: @@ -337,5 +359,8 @@ def change_state(self, ifname, state='up', psid=None): try: self.link.set_state(state=state, ifname=ifname, net_ns_fd=netns_name) - except: - raise UnableToChangeState() + except Exception as e: + msg = ("Unable to change state of %s interface to %s state in " + "%s namespace.ERROR: %s" % (ifname, state, net_ns_fd, e)) + + raise UnableToChangeState(msg) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/linux.py b/dockyard/dockyard-engine/network/drivers/bridges/linux.py index 10fe6fd..9aacd5b 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/linux.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/linux.py @@ -1,35 +1,33 @@ -from pyroute2 import IPRoute - -from base import InterfaceManager, IPManager +from base import InterfaceManager, IPManager, Link from network_driver_exceptions import ( InsufficientInfo, + UnableToAttachPort, UnableToCreateInterface) +from utils import RandomNumber class LinuxBridgeManager(object): + MAX_IF_LENGTH = 15 + def __init__(self): - self.ln = IPRoute() + self.link = Link() self.if_manager = InterfaceManager() self.ip_manager = IPManager() - def _get_ifname(self, id_, loc, prefix='dockyard'): + def _get_ifname(self, prefix='deth'): """This method makes name of the interface in the specified format for our application. - """ - - if not loc: - msg = "Location of the interface is not defined" - raise InsufficientInfo(msg) - if loc == 'external': - loc = 'x' - elif loc == 'internal': - loc = 'i' - - return ('%s-%s-%s' % (prefix, loc, id_)); + Name returned by this method should not be more than 15 length + character. + """ + + rand_num = RandomNumber() + id_ = rand_num.get_number(self.MAX_IF_LENGTH - (len(prefix) + 1)) + return ('%s-%s' % (prefix, id_)); - def create_link_pair(self, id_, kind='veth', peer=None): + def create_link_pair(self, ifname=None, kind='veth', peer=None): """Creates links, one for namespace and other out of namespace. ifname: Name of the interface. In this application, it will @@ -42,30 +40,35 @@ def create_link_pair(self, id_, kind='veth', peer=None): link and '[kind]-i-[id]' for the namespace link. """ - ext_if = self._get_ifname(id_, loc='external') + if not ifname: + ext_if = self._get_ifname() + else: + ext_if = ifname if not peer: - int_if = self._get_ifname(id_, loc='internal') + int_if = self._get_ifname() try: - self.ln.link('add', - ifname=ext_if, - kind=kind, - peer=int_if) - except: - msg = ("Unable to create %s, %s interfaces of %s kind" % ( - ext_if, int_if, kind)) + self.link.create(ifname=ext_if, kind=kind, peer=int_if) + except Exception as e: + msg = ("Unable to create %s, %s interfaces of %s kind. ERROR: %s" + % (ext_if, int_if, kind, e)) raise UnableToCreateInterface(msg) return {'ext_if': ext_if, 'int_if': int_if} # Make This method working - def attach_if(self, ifname, br_name): + def attach_port(self, ifname, br_name): """Attach interface to bridges. """ - self.ln.link('set', index=ifname, - master=br_name) + try: + self.link.attach_port(ifname=ifname, bridge=br_name) + except Exception as e: + msg = ("Unable to attach %s interface with %s bridge. ERROR: %s" + % (ifname, bridge, e)) + + raise UnableToAttachPort(msg) def move_to_namespace(self, ifname, psid): """move an interface to the docker process namespace. diff --git a/dockyard/dockyard-engine/network/drivers/bridges/namespace.py b/dockyard/dockyard-engine/network/drivers/bridges/namespace.py index aa21884..1fa4bb5 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/namespace.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/namespace.py @@ -70,3 +70,9 @@ def get_netns_name(self, psid): """ netns_name = ("dockyard_%d" % (psid)) return netns_name + + def cleanup(self, psid): + """cleanup namespace by removing link to it. + """ + netns_loc = self._get_netns_loc(psid) + self.symlink.cleanup(netns_loc) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py b/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py index 42a7d4b..111b555 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py @@ -2,10 +2,12 @@ class BridgeDriversExceptions(Exception): pass - class NamespaceNotFound(BridgeDriversExceptions): pass +class UnableToAttachPort(BridgeDriversExceptions): + pass + class UnableToAddRoutes(BridgeDriversExceptions): pass diff --git a/dockyard/dockyard-engine/network/drivers/bridges/symlink.py b/dockyard/dockyard-engine/network/drivers/bridges/symlink.py index 5226252..789df0c 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/symlink.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/symlink.py @@ -17,3 +17,9 @@ def is_symlink(self, path): """This method checks whether given path is symlink or not. """ return os.path.islink(path) + + def cleanup(self, path): + """Remove the file from a path. + :path: Path of the file to remove. + """ + os.remove(path) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_add_port.py b/dockyard/dockyard-engine/network/drivers/bridges/test_add_port.py new file mode 100644 index 0000000..c605b00 --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/test_add_port.py @@ -0,0 +1,3 @@ +from linux import LinuxBridgeManager +obj = LinuxBridgeManager() +obj.attach_if('deth-QXQZrnifAQ', 'br100') diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_cleanup.py b/dockyard/dockyard-engine/network/drivers/bridges/test_cleanup.py new file mode 100644 index 0000000..02bb23b --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/test_cleanup.py @@ -0,0 +1,6 @@ +from namespace import DockyardNamespace +psid=[2057] +ns = DockyardNamespace() +for pid in psid: + ns.cleanup(pid) + diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_linux.py b/dockyard/dockyard-engine/network/drivers/bridges/test_linux.py index c086f80..704e6be 100644 --- a/dockyard/dockyard-engine/network/drivers/bridges/test_linux.py +++ b/dockyard/dockyard-engine/network/drivers/bridges/test_linux.py @@ -1,11 +1,13 @@ from linux import LinuxBridgeManager +#from namespace import DockyardNamspace obj = LinuxBridgeManager() -psid=4067 -ifs = {'int_if': ('dockyard-i-%d' % psid), - 'ext_if': ('dockyard-x-%d' % psid)} -obj.create_link_pair(psid) +#ns = DockyardNamespace() +psid=8074 +ifs = obj.create_link_pair() +obj.attach_port(ifname=ifs['ext_if'], br_name='br100') obj.move_to_namespace(ifname=ifs['int_if'], psid=psid) obj.change_state(ifname=ifs['int_if'], state='up', psid=psid) -obj.addr(ifname=ifs['int_if'], address='192.168.100.4', mask=24, psid=psid) +obj.addr(ifname=ifs['int_if'], address='192.168.100.7', mask=24, psid=psid) obj.add_routes(dst='default', gateway='192.168.100.1', psid=psid, oif_name=ifs['int_if']) +#ns.cleanup(psid) diff --git a/dockyard/dockyard-engine/network/drivers/bridges/utils.py b/dockyard/dockyard-engine/network/drivers/bridges/utils.py new file mode 100644 index 0000000..eeb2829 --- /dev/null +++ b/dockyard/dockyard-engine/network/drivers/bridges/utils.py @@ -0,0 +1,32 @@ +import abc +import random +import string + +class Random(object): + def __init__(self): + assert(False) + + @abc.abstractmethod + def get_number(self, length): + """This number generate a random string from of given + length. + """ + +class RandomNumber(Random): + """This class generate random string. + """ + def __init__(self): + pass + + def get_number(self, length): + """This method returns a random string of length. + random strings can be in lower case, upper case + and digits. + """ + choices = (string.ascii_lowercase + string.ascii_uppercase + + string.digits) + + rand = (''.join(random.SystemRandom().choice(choices) + for _ in range(length))) + + return rand From 4081893e89640af6d82fb42ba91a3f56d1f37652 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 21 May 2016 16:46:55 +0530 Subject: [PATCH 38/62] Patch for Issue 22 --- dockyard/common/base.py | 62 +++++++++++++++++++ dockyard/common/utils.py | 46 +++++++++----- dockyard/controllers/v1/__init__.py | 15 +++++ .../{dockyard-engine => engine}/__init__.py | 0 .../common}/network/__init__.py | 0 .../common}/network/drivers/__init__.py | 0 .../network/drivers/bridges/__init__.py | 0 .../common}/network/drivers/bridges/base.py | 0 .../common}/network/drivers/bridges/linux.py | 0 .../network/drivers/bridges/namespace.py | 0 .../bridges/network_driver_exceptions.py | 0 .../network/drivers/bridges/symlink.py | 0 .../network/drivers/bridges/test_add_port.py | 0 .../network/drivers/bridges/test_cleanup.py | 0 .../network/drivers/bridges/test_linux.py | 0 .../common}/network/drivers/bridges/utils.py | 0 dockyard/engine/controllers/__init__.py | 0 dockyard/engine/controllers/interface.py | 43 +++++++++++++ 18 files changed, 151 insertions(+), 15 deletions(-) create mode 100644 dockyard/common/base.py rename dockyard/{dockyard-engine => engine}/__init__.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/__init__.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/__init__.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/bridges/__init__.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/bridges/base.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/bridges/linux.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/bridges/namespace.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/bridges/network_driver_exceptions.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/bridges/symlink.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/bridges/test_add_port.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/bridges/test_cleanup.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/bridges/test_linux.py (100%) rename dockyard/{dockyard-engine => engine/common}/network/drivers/bridges/utils.py (100%) create mode 100644 dockyard/engine/controllers/__init__.py create mode 100644 dockyard/engine/controllers/interface.py diff --git a/dockyard/common/base.py b/dockyard/common/base.py new file mode 100644 index 0000000..7be4934 --- /dev/null +++ b/dockyard/common/base.py @@ -0,0 +1,62 @@ +import abc +from pecan import request +from urllib3 import PoolManager + +class URL(object): + def __init__(self): + pass + + @abc.abstractmethod + def send(self, method, url, headers=None, post_params=None, + body=None, **kwargs): + """This methos is responsible for sending the request. + :method: Method to be used for sending request. + :url: URL to be send. + :headers: headers in the requests. + :post_params: post parameters. + :body: Request body. + """ + +class DockyardURL(URL): + def __init__(self): + self.pool = PoolManager() + + def _is_local_request(self): + """This method checks whether this is request for docker engine. + """ + try: + request.environ.headers['Request-Status'] + except KeyError: + status = False + else: + status = True + + return status + + def send(self, method, url, headers=None, post_params=None, + body=None, **kwargs): + """This methos is responsible for sending the request. + :method: Method to be used for sending request. + :url: URL to be send. + :headers: headers in the requests. + :post_params: post parameters. + :body: Request body. + """ + req_type = self._is_local_request(): + if req_type: + preprocess + else: + headers = self._add_headers(headers) + + data = self.pool.urlopen(method, url, headers=headers, body=body).data + post_process + return data + + + def _add_headers(self, headers): + """This method adds header to each request. + Valid values for Request-Status header are Scheduled. + More values will be added, as per requirements. + """ + headers['Request-Status'] = 'Scheduled' + returns headers diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index 0376e1d..db8ddb1 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -1,7 +1,8 @@ import importlib from oslo_config import cfg -from urllib3 import PoolManager +from pecan import request as rcvd_req +from base import DockyardURL from dockyard.common import base, link SCHEDULER_OPT = [ @@ -45,6 +46,7 @@ membership = class_() membership.register() +request = DockyardURL() def get_config(group, option): CONF = cfg.CONF @@ -52,8 +54,23 @@ def get_config(group, option): def get_host(): - hosts = membership.get_all_hosts() - return scheduler.get_host(hosts=hosts) + """This method returns host, for serving the request. + + If this instance of the process is api server then scheduler + will be used for scheduling host. + + If this instance of the process is engine then it should direct + to docker procss running on local machine and does preprocessing. + """ + try: + rcvd_req.headers.environ['Request-Status'] + except KeyError: + hosts = membership.get_all_hosts() + host = scheduler.get_host(hosts=hosts) + else: + host = { 'host': CONF['docker_host'], 'port': CONF['docker_port'] } + + return host def get_link(url, protocol='http'): @@ -61,15 +78,14 @@ def get_link(url, protocol='http'): return link.make_url(host=host['host'], port=host['port'], url=url) -def dispatch_get_request(url, protocol='http', query_params=None): +def dispatch_get_request(url, headers=None, protocol='http', query_params=None): ln = get_link(url, protocol) - pool = PoolManager() if query_params: query = link.make_query_url(query_params) ln = (('%s?%s') % (ln, query)) - return pool.urlopen('GET', url=ln).data + return request.send(method='GET', url=ln) def dispatch_post_request(url, protocol='http', body=None, query_params=None): @@ -79,7 +95,7 @@ def dispatch_post_request(url, protocol='http', body=None, query_params=None): query = link.make_query_url(query_params) ln = (('%s?%s') % (ln, query)) - return dispatch_post_req(url=ln, post_params=query_params, body=body).data + return dispatch_post_req(url=ln, post_params=query_params, body=body) def dispatch_put_request(url, protocol='http', body=None, query_params=None): @@ -89,24 +105,24 @@ def dispatch_put_request(url, protocol='http', body=None, query_params=None): query = link.make_query_url(query_params) ln = (('%s?%s') % (ln, query)) - return dispatch_put_req(url=ln, post_params=query_params, body=body).data + return dispatch_put_req(url=ln, post_params=query_params, body=body) -def dispatch_delete_request(url, protocol='http', query_params=None): - pool = PoolManager() +def dispatch_delete_request(url, headers = None, protocol='http', + query_params=None): ln = get_link(url, protocol) - return pool.urlopen('DELETE', url=ln) + return request.send(method='DELETE', url=ln) def dispatch_post_req(url, headers=None, body=None, post_params=None): - pool = PoolManager() if not headers: headers = {'Content-Type': 'application/json'} - return pool.urlopen('POST', url, headers=headers, body=body).data + + return request.send(method='POST', url=url, headers=headers, body=body) def dispatch_put_req(url, headers=None, body=None, post_params=None): - pool = PoolManager() if not headers: headers = {'Content-Type': 'application/x-tar'} - return pool.urlopen('PUT', url, headers=headers, body=body).data + + return request.send(method='PUT', url=url, headers=headers, body=body) diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py index 0e7ab6f..d3bec9e 100644 --- a/dockyard/controllers/v1/__init__.py +++ b/dockyard/controllers/v1/__init__.py @@ -6,7 +6,21 @@ from dockyard.controllers.v1 import cluster from pecan import rest from pecan import expose +from oslo_config import cfg +SERVER_OPT = [ + cfg.BoolOpt('api', + default='False', + help=('True for api server, False ' + 'for disabling api server')), + cfg.BoolOpt('engine', + default='True', + help=('True for engine server, ' + 'False for disabling engine server.')) + ] + +CONF = cfg.CONF +CONF.register_opts(SERVER_OPT, group='default') class Controller(rest.RestController): info = information.Information() @@ -16,3 +30,4 @@ class Controller(rest.RestController): networks = network.NetworkController() volumes = volume.VolumeController() clusters = cluster.ClusterController() + diff --git a/dockyard/dockyard-engine/__init__.py b/dockyard/engine/__init__.py similarity index 100% rename from dockyard/dockyard-engine/__init__.py rename to dockyard/engine/__init__.py diff --git a/dockyard/dockyard-engine/network/__init__.py b/dockyard/engine/common/network/__init__.py similarity index 100% rename from dockyard/dockyard-engine/network/__init__.py rename to dockyard/engine/common/network/__init__.py diff --git a/dockyard/dockyard-engine/network/drivers/__init__.py b/dockyard/engine/common/network/drivers/__init__.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/__init__.py rename to dockyard/engine/common/network/drivers/__init__.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/__init__.py b/dockyard/engine/common/network/drivers/bridges/__init__.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/bridges/__init__.py rename to dockyard/engine/common/network/drivers/bridges/__init__.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/base.py b/dockyard/engine/common/network/drivers/bridges/base.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/bridges/base.py rename to dockyard/engine/common/network/drivers/bridges/base.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/linux.py b/dockyard/engine/common/network/drivers/bridges/linux.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/bridges/linux.py rename to dockyard/engine/common/network/drivers/bridges/linux.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/namespace.py b/dockyard/engine/common/network/drivers/bridges/namespace.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/bridges/namespace.py rename to dockyard/engine/common/network/drivers/bridges/namespace.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py b/dockyard/engine/common/network/drivers/bridges/network_driver_exceptions.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/bridges/network_driver_exceptions.py rename to dockyard/engine/common/network/drivers/bridges/network_driver_exceptions.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/symlink.py b/dockyard/engine/common/network/drivers/bridges/symlink.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/bridges/symlink.py rename to dockyard/engine/common/network/drivers/bridges/symlink.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_add_port.py b/dockyard/engine/common/network/drivers/bridges/test_add_port.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/bridges/test_add_port.py rename to dockyard/engine/common/network/drivers/bridges/test_add_port.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_cleanup.py b/dockyard/engine/common/network/drivers/bridges/test_cleanup.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/bridges/test_cleanup.py rename to dockyard/engine/common/network/drivers/bridges/test_cleanup.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/test_linux.py b/dockyard/engine/common/network/drivers/bridges/test_linux.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/bridges/test_linux.py rename to dockyard/engine/common/network/drivers/bridges/test_linux.py diff --git a/dockyard/dockyard-engine/network/drivers/bridges/utils.py b/dockyard/engine/common/network/drivers/bridges/utils.py similarity index 100% rename from dockyard/dockyard-engine/network/drivers/bridges/utils.py rename to dockyard/engine/common/network/drivers/bridges/utils.py diff --git a/dockyard/engine/controllers/__init__.py b/dockyard/engine/controllers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine/controllers/interface.py b/dockyard/engine/controllers/interface.py new file mode 100644 index 0000000..d870e87 --- /dev/null +++ b/dockyard/engine/controllers/interface.py @@ -0,0 +1,43 @@ +from pecan import expose, abort + +class Interface(object): + def __init__(self): + pass + + @expose() + def create(self): + pass + + @expose() + def delete(self): + pass + + @expose() + def attach(self): + pass + + @expose() + def move(self): + pass + + @expose() + def detach(self): + pass + + +class IP() + def __init__(self): + pass + + @expose() + def assign(self): + pass + + +class Routes(object): + def __init__(self): + pass + + @expose + def gateway(self): + pass From f5b6a8bb5a9d87b7a763adafaae391063f64aed5 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Mon, 23 May 2016 22:36:01 +0530 Subject: [PATCH 39/62] Work on Controller for docker engine --- dockyard/engine/controllers/__init__.py | 11 ++++ dockyard/engine/controllers/interface.py | 43 ------------ dockyard/engine/controllers/network.py | 83 ++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 43 deletions(-) delete mode 100644 dockyard/engine/controllers/interface.py create mode 100644 dockyard/engine/controllers/network.py diff --git a/dockyard/engine/controllers/__init__.py b/dockyard/engine/controllers/__init__.py index e69de29..0cdfd6b 100644 --- a/dockyard/engine/controllers/__init__.py +++ b/dockyard/engine/controllers/__init__.py @@ -0,0 +1,11 @@ +from oslo_config import cfg + + +NETWORK_DRIVER_OPT = [ + cfg.StrOpt('network_driver', + default='bridges.linux.LinuxBridgeManager', + help='Network driver for the changing settings for network.') +] + +CONF = cfg.CONF +CONF.register_opts(NETWORK_DRIVER_OPT, group='default') diff --git a/dockyard/engine/controllers/interface.py b/dockyard/engine/controllers/interface.py deleted file mode 100644 index d870e87..0000000 --- a/dockyard/engine/controllers/interface.py +++ /dev/null @@ -1,43 +0,0 @@ -from pecan import expose, abort - -class Interface(object): - def __init__(self): - pass - - @expose() - def create(self): - pass - - @expose() - def delete(self): - pass - - @expose() - def attach(self): - pass - - @expose() - def move(self): - pass - - @expose() - def detach(self): - pass - - -class IP() - def __init__(self): - pass - - @expose() - def assign(self): - pass - - -class Routes(object): - def __init__(self): - pass - - @expose - def gateway(self): - pass diff --git a/dockyard/engine/controllers/network.py b/dockyard/engine/controllers/network.py new file mode 100644 index 0000000..61e865a --- /dev/null +++ b/dockyard/engine/controllers/network.py @@ -0,0 +1,83 @@ +from oslo_config import cfg +from pecan import expose, abort, rest + + +# Fetch scheduler defined in the configuration file and load it. +network_driver_info = CONF.default.network_driver +# May be this path can be specified in the configuration file. +network_driver_loc = 'engine.common.network.drivers' +network_driver_info = (('%s.%s') % (nework_driver_loc, network_driver_info)) +module_name, class_name = scheduler_info.rsplit(".", 1) +class_ = getattr(importlib.import_module(module_name), class_name) +network = class_() + + +class Interface(rest.Controller): + def __init__(self): + pass + + @expose() + def post(self): + pass + + @expose() + def delete(self): + pass + + @expose() + def attach(self): + pass + + @expose() + def move(self): + pass + + @expose() + def detach(self): + pass + + @expose() + def get_all(self): + pass + + @expose() + def get_one(self): + pass + +class Routes(rest.Controller): + def __init__(self): + pass + + @expose() + def post(self): + pass + + def delete(self): + pass + + def put(self): + pass + + def get_one(self): + pass + + +class IP(rest.Controller) + def __init__(self): + pass + + @expose() + def put(self): + pass + + @expose() + def delete(self): + pass + + @expose() + def udpate(self): + pass + + @expose() + def get_one(self): + pass From 94ddbe908a65a52515c9584825cf5866d7e9e196 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Wed, 25 May 2016 23:08:27 +0530 Subject: [PATCH 40/62] API for interface --- dockyard/common/base.py | 11 ++-- dockyard/controllers/v1/__init__.py | 33 ++++++---- dockyard/engine/common/__init__.py | 0 .../common/network/drivers/bridges/linux.py | 41 +++++++++++- dockyard/engine/common/network/manager.py | 51 +++++++++++++++ dockyard/engine/common/utils.py | 6 ++ dockyard/engine/controllers/network.py | 65 ++++++++++++------- 7 files changed, 163 insertions(+), 44 deletions(-) create mode 100644 dockyard/engine/common/__init__.py create mode 100644 dockyard/engine/common/network/manager.py create mode 100644 dockyard/engine/common/utils.py diff --git a/dockyard/common/base.py b/dockyard/common/base.py index 7be4934..c28e20e 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -25,7 +25,7 @@ def _is_local_request(self): """This method checks whether this is request for docker engine. """ try: - request.environ.headers['Request-Status'] + request.headers.environ['Request-Status'] except KeyError: status = False else: @@ -42,14 +42,14 @@ def send(self, method, url, headers=None, post_params=None, :post_params: post parameters. :body: Request body. """ - req_type = self._is_local_request(): + req_type = self._is_local_request() if req_type: preprocess else: headers = self._add_headers(headers) data = self.pool.urlopen(method, url, headers=headers, body=body).data - post_process + # post_process return data @@ -58,5 +58,8 @@ def _add_headers(self, headers): Valid values for Request-Status header are Scheduled. More values will be added, as per requirements. """ + if not headers: + headers = dict() + headers['Request-Status'] = 'Scheduled' - returns headers + return headers diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py index d3bec9e..98cef76 100644 --- a/dockyard/controllers/v1/__init__.py +++ b/dockyard/controllers/v1/__init__.py @@ -1,16 +1,19 @@ +from pecan import rest +from pecan import expose +from oslo_config import cfg + from dockyard.controllers.v1 import information from dockyard.controllers.v1 import container from dockyard.controllers.v1 import image from dockyard.controllers.v1 import network from dockyard.controllers.v1 import volume from dockyard.controllers.v1 import cluster -from pecan import rest -from pecan import expose -from oslo_config import cfg +from dockyard.engine.controllers.network import DockyardEngineController + SERVER_OPT = [ cfg.BoolOpt('api', - default='False', + default='True', help=('True for api server, False ' 'for disabling api server')), cfg.BoolOpt('engine', @@ -22,12 +25,18 @@ CONF = cfg.CONF CONF.register_opts(SERVER_OPT, group='default') + class Controller(rest.RestController): - info = information.Information() - version = information.Version() - containers = container.ContainerController() - images = image.ImageController() - networks = network.NetworkController() - volumes = volume.VolumeController() - clusters = cluster.ClusterController() - + """Controller for both the api server and engine. + """ + if CONF.default.api: + info = information.Information() + version = information.Version() + containers = container.ContainerController() + images = image.ImageController() + networks = network.NetworkController() + volumes = volume.VolumeController() + clusters = cluster.ClusterController() + + if CONF.default.engine: + dockyard = DockyardEngineController() diff --git a/dockyard/engine/common/__init__.py b/dockyard/engine/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine/common/network/drivers/bridges/linux.py b/dockyard/engine/common/network/drivers/bridges/linux.py index 9aacd5b..12834fb 100644 --- a/dockyard/engine/common/network/drivers/bridges/linux.py +++ b/dockyard/engine/common/network/drivers/bridges/linux.py @@ -1,7 +1,10 @@ from base import InterfaceManager, IPManager, Link +from namespace import DockyardNamespace + from network_driver_exceptions import ( InsufficientInfo, UnableToAttachPort, + NamespaceNotFound, UnableToCreateInterface) from utils import RandomNumber @@ -13,6 +16,7 @@ def __init__(self): self.link = Link() self.if_manager = InterfaceManager() self.ip_manager = IPManager() + self.namespace = DockyardNamespace() def _get_ifname(self, prefix='deth'): """This method makes name of the interface in the specified format @@ -54,11 +58,10 @@ def create_link_pair(self, ifname=None, kind='veth', peer=None): msg = ("Unable to create %s, %s interfaces of %s kind. ERROR: %s" % (ext_if, int_if, kind, e)) - raise UnableToCreateInterface(msg) + return msg return {'ext_if': ext_if, 'int_if': int_if} - # Make This method working def attach_port(self, ifname, br_name): """Attach interface to bridges. """ @@ -112,8 +115,40 @@ def add_routes(self, oif_name, dst='default', gateway='0.0.0.0', :kwargs: In case of advanced networking, additional parameters might be provided through this option. """ + netns_name = None + if psid: - self.if_manager.netns.get_netns_name(psid=psid) + netns_name = self.if_manager.netns.get_netns_name(psid=psid) self.ip_manager.add_routes(oif_name=oif_name, dst=dst, gateway=gateway, psid=psid, **kwargs) + + def get_ifs(self, psid=None): + """This method returns all the network interfaces defined in docker + network namespace. + """ + netns_name = None + + if psid: + netns_name = self.if_manager.netns.get_netns_name(psid=psid) + + try: + self._check_and_attach(psid) + except NamespaceNotFound as e: + # 404 into the header, which suggest on client side + # about this issue. + return ("Namespace %d could not be found. ERROR: %s" % (psid, e)) + + return self.link.get_ifs(net_ns_fd=netns_name) + + # This method has been defined in at 3 places, it must be put into + # namespace class so that it can be reused at all the places. + def _check_and_attach(self, psid): + if not self.namespace.does_exist(psid): + try: + self.namespace.attach_namespace(psid) + except Exception as e: + msg = ("%s Namespace does not exist. ERROR: %s" % (psid, e)) + raise NamespaceNotFound(msg) + + return psid diff --git a/dockyard/engine/common/network/manager.py b/dockyard/engine/common/network/manager.py new file mode 100644 index 0000000..4157cc4 --- /dev/null +++ b/dockyard/engine/common/network/manager.py @@ -0,0 +1,51 @@ +import importlib +import json +from oslo_config import cfg + +from dockyard.engine.common.utils import json_dump + +CONF = cfg.CONF +# Fetch scheduler defined in the configuration file and load it. +network_driver_info = CONF.default.network_driver +# May be this path can be specified in the configuration file. +network_driver_loc = 'dockyard.engine.common.network.drivers' +network_driver_info = (('%s.%s') % (network_driver_loc, network_driver_info)) +module_name, class_name = network_driver_info.rsplit(".", 1) +class_ = getattr(importlib.import_module(module_name), class_name) + +class NetworkManager(object): + """This class acts as middleware between drivers and requests. + """ + def __init__(self): + self.network = class_() + + def get_ifs(self, psid): + """This method returns all the interfaces of the docker container. + :params psid: process id the container. + docker inspect | grep Pid + + :returns: Number of interface in a docker container. + """ + if not psid: + msg = ("Invalid namespace, Namespace can not be None.") + return msg + + psid = int(psid) + ifs = dict() + ifs[psid] = self.network.get_ifs(psid) + return json_dump(ifs) + + def create(self, ifname=None, kind='veth', peer=None): + """This method creates interfaces in the namespace. + :params ifname: Interface name, if not defined random + string will be chossen. + :params kind: Kind of the veth pair. + :params peer: peer interface name for this interface. + """ + ifs_names = dict() + ifs_names['interface_names'] = \ + self.network.create_link_pair(ifname=ifname, + kind=kind, peer=peer) + + return json_dump(ifs_names) + diff --git a/dockyard/engine/common/utils.py b/dockyard/engine/common/utils.py new file mode 100644 index 0000000..3d603cd --- /dev/null +++ b/dockyard/engine/common/utils.py @@ -0,0 +1,6 @@ +import json + +def json_dump(data): + """This function dumps data into json format to send over the internet. + """ + return json.dumps(data) diff --git a/dockyard/engine/controllers/network.py b/dockyard/engine/controllers/network.py index 61e865a..86244da 100644 --- a/dockyard/engine/controllers/network.py +++ b/dockyard/engine/controllers/network.py @@ -1,24 +1,18 @@ -from oslo_config import cfg -from pecan import expose, abort, rest +from pecan import expose, abort, request +from pecan.rest import RestController +from dockyard.engine.common.network.manager import NetworkManager -# Fetch scheduler defined in the configuration file and load it. -network_driver_info = CONF.default.network_driver -# May be this path can be specified in the configuration file. -network_driver_loc = 'engine.common.network.drivers' -network_driver_info = (('%s.%s') % (nework_driver_loc, network_driver_info)) -module_name, class_name = scheduler_info.rsplit(".", 1) -class_ = getattr(importlib.import_module(module_name), class_name) -network = class_() - - -class Interface(rest.Controller): +class InterfaceController(RestController): def __init__(self): - pass + self.network = NetworkManager() @expose() def post(self): - pass + """This method is responsible for creating network interfaces. + """ + body = request.body + return self.network.create(body) @expose() def delete(self): @@ -36,15 +30,14 @@ def move(self): def detach(self): pass - @expose() - def get_all(self): - pass - - @expose() - def get_one(self): - pass + # created this for testing purpose only +# @expose() + def get_one(self, psid): + """This method returns networks interfaces in a container. + """ + return self.network.get_ifs(psid=psid) -class Routes(rest.Controller): +class RoutesController(RestController): def __init__(self): pass @@ -62,7 +55,7 @@ def get_one(self): pass -class IP(rest.Controller) +class IPController(RestController): def __init__(self): pass @@ -79,5 +72,27 @@ def udpate(self): pass @expose() - def get_one(self): + def get_one(self, psid=None): + """This method returns IPs allocated to a container. + """ + pass + + +class DockyardEngineController(RestController): + """Controller for the Engine. + """ + + """This is the controller for the interface. + """ + interface = InterfaceController() + + """Controller for IPs. + """ + ip = IPController() + + """Controller for routes. + """ + routes = RoutesController() + + def __init__(self): pass From b6fe20e399eb45b89ac506666c0c70a8e16b0346 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Thu, 26 May 2016 22:29:21 +0530 Subject: [PATCH 41/62] APIs for interface --- dockyard/engine/common/exceptions.py | 2 + .../common/network/drivers/bridges/linux.py | 2 + dockyard/engine/common/network/manager.py | 40 +++++++++++++++---- dockyard/engine/common/utils.py | 6 +++ dockyard/engine/controllers/network.py | 15 ++++--- 5 files changed, 50 insertions(+), 15 deletions(-) create mode 100644 dockyard/engine/common/exceptions.py diff --git a/dockyard/engine/common/exceptions.py b/dockyard/engine/common/exceptions.py new file mode 100644 index 0000000..0acfa23 --- /dev/null +++ b/dockyard/engine/common/exceptions.py @@ -0,0 +1,2 @@ +class InterfaceNotFound(object): + msg = ("Interface not found.") diff --git a/dockyard/engine/common/network/drivers/bridges/linux.py b/dockyard/engine/common/network/drivers/bridges/linux.py index 12834fb..dc9abbe 100644 --- a/dockyard/engine/common/network/drivers/bridges/linux.py +++ b/dockyard/engine/common/network/drivers/bridges/linux.py @@ -51,6 +51,8 @@ def create_link_pair(self, ifname=None, kind='veth', peer=None): if not peer: int_if = self._get_ifname() + else: + int_if = peer try: self.link.create(ifname=ext_if, kind=kind, peer=int_if) diff --git a/dockyard/engine/common/network/manager.py b/dockyard/engine/common/network/manager.py index 4157cc4..f22f4a3 100644 --- a/dockyard/engine/common/network/manager.py +++ b/dockyard/engine/common/network/manager.py @@ -1,8 +1,8 @@ import importlib -import json from oslo_config import cfg -from dockyard.engine.common.utils import json_dump +from dockyard.engine.common.utils import json_dump, str_to_dict +from dockyard.engine.common.exceptions import InterfaceNotFound CONF = cfg.CONF # Fetch scheduler defined in the configuration file and load it. @@ -13,6 +13,7 @@ module_name, class_name = network_driver_info.rsplit(".", 1) class_ = getattr(importlib.import_module(module_name), class_name) + class NetworkManager(object): """This class acts as middleware between drivers and requests. """ @@ -30,12 +31,11 @@ def get_ifs(self, psid): msg = ("Invalid namespace, Namespace can not be None.") return msg - psid = int(psid) ifs = dict() ifs[psid] = self.network.get_ifs(psid) return json_dump(ifs) - def create(self, ifname=None, kind='veth', peer=None): + def create(self, peer=None, ifname=None, kind='veth'): """This method creates interfaces in the namespace. :params ifname: Interface name, if not defined random string will be chossen. @@ -44,8 +44,34 @@ def create(self, ifname=None, kind='veth', peer=None): """ ifs_names = dict() ifs_names['interface_names'] = \ - self.network.create_link_pair(ifname=ifname, - kind=kind, peer=peer) + self.network.create_link_pair(ifname=ifname, peer=peer, + kind=kind) return json_dump(ifs_names) - + + def update(self, ifname, brname=None, psid=None, state=None): + """This method is responsible for attaching network interface to + a bridge or moving an interface to a namespace. + """ + if brname and psid and state: + msg = ("Operation not supprted") + return json_dump(msg) + + data = dict() + if brname: + data['port_info'] = self.network.attach_port(ifname=ifname, + br_name=brname) + else: + if psid: + psid = int(psid) + try: + data["interface_info"] = self.network.move_to_namespace( + ifname=ifname, psid=psid) + except: + pass + + if state: + data["state"] = self.network.change_state(ifname=ifname, psid=psid, + state=state) + + return json_dump(data) diff --git a/dockyard/engine/common/utils.py b/dockyard/engine/common/utils.py index 3d603cd..ff9f9f3 100644 --- a/dockyard/engine/common/utils.py +++ b/dockyard/engine/common/utils.py @@ -1,6 +1,12 @@ +import ast import json def json_dump(data): """This function dumps data into json format to send over the internet. """ return json.dumps(data) + +def str_to_dict(data): + """This function convert string to dict. + """ + return ast.literal_eval(data) diff --git a/dockyard/engine/controllers/network.py b/dockyard/engine/controllers/network.py index 86244da..0ea1a08 100644 --- a/dockyard/engine/controllers/network.py +++ b/dockyard/engine/controllers/network.py @@ -1,6 +1,7 @@ from pecan import expose, abort, request from pecan.rest import RestController +from dockyard.engine.common.utils import str_to_dict from dockyard.engine.common.network.manager import NetworkManager class InterfaceController(RestController): @@ -11,20 +12,17 @@ def __init__(self): def post(self): """This method is responsible for creating network interfaces. """ - body = request.body - return self.network.create(body) + kwargs = str_to_dict(request.body) + return self.network.create(**kwargs) @expose() def delete(self): pass @expose() - def attach(self): - pass - - @expose() - def move(self): - pass + def put(self): + kwargs = str_to_dict(request.body) + return self.network.update(**kwargs) @expose() def detach(self): @@ -35,6 +33,7 @@ def detach(self): def get_one(self, psid): """This method returns networks interfaces in a container. """ + psid = int(psid) return self.network.get_ifs(psid=psid) class RoutesController(RestController): From d6db3c4a4e1abd32a3a3ee5fbb8b6842bed9bf99 Mon Sep 17 00:00:00 2001 From: karelyatin Date: Sat, 28 May 2016 17:42:27 +0530 Subject: [PATCH 42/62] included oslo logging against issue #6 --- bugs/new_blueprints.txt | 1 + dockyard/cmd/api.py | 15 +++++++++++++++ dockyard/common/cluster/cluster.py | 2 ++ dockyard/common/container/container.py | 7 +++++++ dockyard/common/image/image.py | 2 +- dockyard/common/information/information.py | 2 +- .../common/membership/consul/consul_driver.py | 4 ++++ dockyard/common/utils.py | 15 ++++++++++++++- dockyard/common/volume/volume.py | 2 +- dockyard/controllers/root.py | 13 ++++++++++++- .../common/network/drivers/bridges/base.py | 16 ++++++++++++---- .../common/network/drivers/bridges/linux.py | 6 +++++- 12 files changed, 75 insertions(+), 10 deletions(-) diff --git a/bugs/new_blueprints.txt b/bugs/new_blueprints.txt index e7d6303..b4c9b06 100755 --- a/bugs/new_blueprints.txt +++ b/bugs/new_blueprints.txt @@ -1,3 +1,4 @@ ======================================================================== Add blueprints that needs to implmented. ======================================================================== +1. Monitoring of containers. User can specify which container to monitor. The monitoring module would monitor the containers and start the stopped container. diff --git a/dockyard/cmd/api.py b/dockyard/cmd/api.py index 818df60..8f11839 100644 --- a/dockyard/cmd/api.py +++ b/dockyard/cmd/api.py @@ -1,15 +1,30 @@ +import os +import sys + from oslo_config import cfg +from oslo_log import log as logging from wsgiref import simple_server from dockyard.api import app as api_app +from dockyard.common import utils + +LOG = logging.getLogger(__name__) def main(): + utils.prepare_logging(sys.argv) + app = api_app.setup_app() # create the wsgi server and start it host, port = cfg.CONF.default.host, cfg.CONF.default.port srv = simple_server.make_server(host, port, app) + + LOG.info('Starting dockyard in PID %s', os.getpid()) + LOG.debug("Configuration:") + cfg.CONF.log_opt_values(LOG, logging.DEBUG) + + LOG.info('serving at http://%s:%s' % (host, port)) srv.serve_forever() diff --git a/dockyard/common/cluster/cluster.py b/dockyard/common/cluster/cluster.py index b83764f..fbb51f9 100644 --- a/dockyard/common/cluster/cluster.py +++ b/dockyard/common/cluster/cluster.py @@ -1,4 +1,5 @@ from oslo_config import cfg +from oslo_log import log as logging class Cluster(object): @@ -12,4 +13,5 @@ def unregister(self, cluster_id, host_ip): pass def get_hosts(self): + LOG.debug("Registered hosts: %s" cfg.CONF.membership.hosts) return cfg.CONF.membership.hosts diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index d0353c0..76fc5da 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -1,9 +1,12 @@ +from oslo_log import log as logging + from dockyard.common import utils, link from dockyard.common import url import json from pecan.core import redirect +LOG = logging.getLogger(__name__) class Container(object): base_url = '/containers' @@ -13,6 +16,10 @@ def __init__(self): def list(self, name_or_id=None, query_params=None): url_ = self.url.make_url(url_='json', id_=name_or_id) + msg = ("List containers with url: %s " + "query_params: %s" % + (url_, query_params)) + LOG.debug(msg) return utils.dispatch_get_request(url_, query_params=query_params) def changes(self, name_or_id=None): diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py index f451c95..5b53ec1 100644 --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -1,5 +1,5 @@ from pecan import abort -from dockyard.common import base, link +from dockyard.common import link from dockyard.common import utils, url diff --git a/dockyard/common/information/information.py b/dockyard/common/information/information.py index 063b9ac..fb25b1a 100644 --- a/dockyard/common/information/information.py +++ b/dockyard/common/information/information.py @@ -1,4 +1,4 @@ -from dockyard.common import base, link +from dockyard.common import link from dockyard.common import utils, url diff --git a/dockyard/common/membership/consul/consul_driver.py b/dockyard/common/membership/consul/consul_driver.py index e4071b1..f8fb1f6 100644 --- a/dockyard/common/membership/consul/consul_driver.py +++ b/dockyard/common/membership/consul/consul_driver.py @@ -1,11 +1,13 @@ import consul from oslo_config import cfg +from oslo_log import log as logging from dockyard.common.membership.base import Membership from dockyard.common import exception CONF = cfg.CONF +LOG = logging.getLogger(__name__) class Consul(Membership): def __init__(self): @@ -14,6 +16,7 @@ def __init__(self): def _register_service(self, name, host, port, tags=None): if not name: message = ('Service name to use for registering') + LOG.exception("Cannot continue, Incomplete info: %s" % message) raise exception.IncompleteInfo(message) # if not host: @@ -22,6 +25,7 @@ def _register_service(self, name, host, port, tags=None): if not port: message = ('Port number used by the services to listen') + LOG.exception("Cannot continue, Incomplete info: %s" % message) raise exception.Incompleteinfo(message) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index db8ddb1..81d73f0 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -1,9 +1,12 @@ import importlib from oslo_config import cfg +from oslo_log import log as logging from pecan import request as rcvd_req from base import DockyardURL -from dockyard.common import base, link +from urllib3 import PoolManager + +from dockyard.common import link SCHEDULER_OPT = [ cfg.StrOpt('scheduler', @@ -77,6 +80,16 @@ def get_link(url, protocol='http'): host = get_host() return link.make_url(host=host['host'], port=host['port'], url=url) +def prepare_logging(argv=None): + """ + log file can be specified as a command line argument + with key = log-file + """ + if argv is None: + argv = [] + logging.register_options(CONF) + CONF(argv[1:], project='dockyard') + logging.setup(CONF, 'dockyard') def dispatch_get_request(url, headers=None, protocol='http', query_params=None): ln = get_link(url, protocol) diff --git a/dockyard/common/volume/volume.py b/dockyard/common/volume/volume.py index 432dd68..fbb0a12 100644 --- a/dockyard/common/volume/volume.py +++ b/dockyard/common/volume/volume.py @@ -1,5 +1,5 @@ import json -from dockyard.common import base, link +from dockyard.common import link from dockyard.common import utils, url diff --git a/dockyard/controllers/root.py b/dockyard/controllers/root.py index d88b9af..cc21fd0 100644 --- a/dockyard/controllers/root.py +++ b/dockyard/controllers/root.py @@ -1,7 +1,10 @@ -from pecan import expose, rest +from oslo_log import log as logging + +from pecan import expose, rest, request from dockyard.controllers import v1 +LOG = logging.getLogger(__name__) class RootController(rest.RestController): @@ -27,4 +30,12 @@ def _route(self, args): if args[0] and args[0] not in self._version: args = [self._default_version] + args + if request.body: + msg = ("Processing request: url: %(url)s, " + "method: %(method)s, " + "body: %(body)s" % + {'url': request.url, + 'method': request.method, + 'body': request.body}) + LOG.debug(msg) return super(RootController, self)._route(args) diff --git a/dockyard/engine/common/network/drivers/bridges/base.py b/dockyard/engine/common/network/drivers/bridges/base.py index b9753de..a33173e 100644 --- a/dockyard/engine/common/network/drivers/bridges/base.py +++ b/dockyard/engine/common/network/drivers/bridges/base.py @@ -3,6 +3,8 @@ # 1) Create Virtual Interface # 2) Moving Virtual Interface into docker namespace # 3) Assign IP address to the interface # 4) Brings up network interface import abc + +from oslo_log import log as logging from network_driver_exceptions import ( AlreadyInNamespace, FailedToMoveInterface, InterfaceNotFound, @@ -15,6 +17,8 @@ from pyroute2 import IPDB, NetNS from namespace import DockyardNamespace + +LOG = logging.getLogger(__name__) # make sure this files interfaces are updated as per # changes. class BridgeManager(object): @@ -216,6 +220,7 @@ def _check_and_attach(self, psid): self.netns.attach_namespace(psid) except Exception as e: msg = ("%s Namespace does not exist. ERROR: %s" % (psid, e)) + LOG.exception(msg) raise NamespaceNotFound(msg) return psid @@ -243,6 +248,8 @@ def assign_ip(self, ifname, address, mask, broadcast=None, psid=None): msg = ("Unable to assign ip %s to %s interface in %s namespace." "ERROR: %s" % (address, ifname, net_ns_fd, e)) + LOG.exception(msg) + raise UnableToAssignIP(msg) def add_routes(self, dst, gateway, oif_name, psid=None, **kwargs): @@ -279,7 +286,7 @@ def add_routes(self, dst, gateway, oif_name, psid=None, **kwargs): msg = ("Unable to add gateway %s for destination %s in namespace " "%s for interface %d. ERROR: %s" % (gateway, dst, netns_name, oif_idx, e)) - + LOG.exception(msg) raise UnableToAddRoutes(msg) @@ -325,6 +332,7 @@ def move_to_namespace(self, ifname, psid): self.netns.attach_namespace(psid) except Exception as e: msg = ("%s Namespace does not exist. ERROR: %s" % (ifname, e)) + LOG.exception(msg) raise NamespaceNotFound(msg) try: @@ -334,7 +342,7 @@ def move_to_namespace(self, ifname, psid): except Exception as e: msg = ("Failed to move %s interface in %s namespace. ERROR: %s" % ( ifname, netns_name, e)) - + LOG.exception(msg) raise FailedToMoveInterface(msg) def change_state(self, ifname, state='up', psid=None): @@ -348,7 +356,7 @@ def change_state(self, ifname, state='up', psid=None): if state not in self.link.allowed_states: msg = ("States has to be among %s but Received %s state" % ( self.link.allowed_states, state)) - + LOG.exception(msg) raise InvalidState(msg) if psid: @@ -362,5 +370,5 @@ def change_state(self, ifname, state='up', psid=None): except Exception as e: msg = ("Unable to change state of %s interface to %s state in " "%s namespace.ERROR: %s" % (ifname, state, net_ns_fd, e)) - + LOG.exception(msg) raise UnableToChangeState(msg) diff --git a/dockyard/engine/common/network/drivers/bridges/linux.py b/dockyard/engine/common/network/drivers/bridges/linux.py index dc9abbe..92381f5 100644 --- a/dockyard/engine/common/network/drivers/bridges/linux.py +++ b/dockyard/engine/common/network/drivers/bridges/linux.py @@ -1,3 +1,4 @@ +from oslo_log import log as logging from base import InterfaceManager, IPManager, Link from namespace import DockyardNamespace @@ -8,6 +9,7 @@ UnableToCreateInterface) from utils import RandomNumber +LOG = logging.getLogger(__name__) class LinuxBridgeManager(object): MAX_IF_LENGTH = 15 @@ -60,6 +62,7 @@ def create_link_pair(self, ifname=None, kind='veth', peer=None): msg = ("Unable to create %s, %s interfaces of %s kind. ERROR: %s" % (ext_if, int_if, kind, e)) + LOG.exception(msg) return msg return {'ext_if': ext_if, 'int_if': int_if} @@ -69,10 +72,11 @@ def attach_port(self, ifname, br_name): """ try: self.link.attach_port(ifname=ifname, bridge=br_name) + LOG.info("Attached interface: %s to bridge: %s" % (ifname, bridge)) except Exception as e: msg = ("Unable to attach %s interface with %s bridge. ERROR: %s" % (ifname, bridge, e)) - + LOG.exception(msg) raise UnableToAttachPort(msg) def move_to_namespace(self, ifname, psid): From d53e54644ca23ebcee09a7491eac86694e8dd3fc Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sat, 28 May 2016 13:11:25 -0400 Subject: [PATCH 43/62] Additing API's for addition of IP address. --- dockyard/engine/common/network/manager.py | 23 ++++++++++++++++++++++- dockyard/engine/controllers/network.py | 13 +++++-------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/dockyard/engine/common/network/manager.py b/dockyard/engine/common/network/manager.py index f22f4a3..984ab08 100644 --- a/dockyard/engine/common/network/manager.py +++ b/dockyard/engine/common/network/manager.py @@ -14,7 +14,7 @@ class_ = getattr(importlib.import_module(module_name), class_name) -class NetworkManager(object): +class IFManager(object): """This class acts as middleware between drivers and requests. """ def __init__(self): @@ -75,3 +75,24 @@ def update(self, ifname, brname=None, psid=None, state=None): state=state) return json_dump(data) + +class IPManager(object): + def __init__(self): + self.ip = class_() + + def addr(self, ifname, address, mask, + psid, broadcast=None): + """This method assigns name in the network namespace of container. + :params ifname: network interface name + :params address: ip address to be assigned + :params mask: netmask for the network + :params psid: network namespace for the container. + :params broadcast: broadcast address + + :returns: returns. + """ + ips = dict() + ips['IP'] = self.ip.addr(ifname=ifname, address=address, mask=mask, + psid=psid, broadcast=broadcast) + + return json_dump(ips) diff --git a/dockyard/engine/controllers/network.py b/dockyard/engine/controllers/network.py index 0ea1a08..78eb143 100644 --- a/dockyard/engine/controllers/network.py +++ b/dockyard/engine/controllers/network.py @@ -2,11 +2,11 @@ from pecan.rest import RestController from dockyard.engine.common.utils import str_to_dict -from dockyard.engine.common.network.manager import NetworkManager +from dockyard.engine.common.network.manager import IFManager, IPManager class InterfaceController(RestController): def __init__(self): - self.network = NetworkManager() + self.network = IFManager() @expose() def post(self): @@ -56,20 +56,17 @@ def get_one(self): class IPController(RestController): def __init__(self): - pass + self.ip = IPManager() @expose() def put(self): - pass + kwargs = str_to_dict(request.body) + return self.ip.addr(**kwargs) @expose() def delete(self): pass - @expose() - def udpate(self): - pass - @expose() def get_one(self, psid=None): """This method returns IPs allocated to a container. From 8b1bcffb5304231cc68188a39f29766ee15cd510 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sun, 29 May 2016 08:31:02 +0530 Subject: [PATCH 44/62] Adding preprocessor and post processor for requests --- bugs/fixed_bugs.txt | 0 bugs/implemented_blueprints.txt | 0 bugs/known_bugs.txt | 15 +++- bugs/new_blueprints.txt | 0 dockyard/common/base.py | 29 ++++++- .../common/network/drivers/bridges/base.py | 4 +- .../common/network/drivers/bridges/linux.py | 1 - dockyard/engine/common/network/manager.py | 75 +++++++++++++++++-- dockyard/engine/controllers/network.py | 71 +++++++++++++----- dockyard/engine/controllers/utils.py | 6 ++ dockyard/engine_client/__init__.py | 0 .../engine_client/api_server/api_server.py | 27 +++++++ dockyard/engine_client/base.py | 20 +++++ test/resize | 0 test/test_engine.sh | 10 +++ test/update | 0 16 files changed, 225 insertions(+), 33 deletions(-) mode change 100755 => 100644 bugs/fixed_bugs.txt mode change 100755 => 100644 bugs/implemented_blueprints.txt mode change 100755 => 100644 bugs/known_bugs.txt mode change 100755 => 100644 bugs/new_blueprints.txt create mode 100644 dockyard/engine/controllers/utils.py create mode 100644 dockyard/engine_client/__init__.py create mode 100644 dockyard/engine_client/api_server/api_server.py create mode 100644 dockyard/engine_client/base.py mode change 100755 => 100644 test/resize create mode 100644 test/test_engine.sh mode change 100755 => 100644 test/update diff --git a/bugs/fixed_bugs.txt b/bugs/fixed_bugs.txt old mode 100755 new mode 100644 diff --git a/bugs/implemented_blueprints.txt b/bugs/implemented_blueprints.txt old mode 100755 new mode 100644 diff --git a/bugs/known_bugs.txt b/bugs/known_bugs.txt old mode 100755 new mode 100644 index 8536e76..d4fa5dc --- a/bugs/known_bugs.txt +++ b/bugs/known_bugs.txt @@ -1,6 +1,15 @@ ================================================================= Add known bugs in this file. Select a format to add bugs. ================================================================= -1) Add feature of receiving IP address from the DHCP -2) Add attach interface function attaching interface to the bridge -3) Method for Communication among docker processes running among cluster hosts +1) Add support for different communication mechnism between api server + and engine. + --> Message queue + --> API server itself + +2) Create /var/run/netns at the start of the dockyard. +3) Add database feature to keep data even after reboot. +4) Add feature to create resource as dockyard start. +5) Write a single function, which will load modules dynmically by taking + information from the configuration file. It is being done many times in + the project which will lead to resuability of the code. +6) Create an API for attaching external network to the docker container diff --git a/bugs/new_blueprints.txt b/bugs/new_blueprints.txt old mode 100755 new mode 100644 diff --git a/dockyard/common/base.py b/dockyard/common/base.py index c28e20e..d01b041 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -2,9 +2,27 @@ from pecan import request from urllib3 import PoolManager +ENGINE_CLIENT_OPT = [ + cfg.StrOpt('engine_client', + default='api_server.api_server.APIServerEngineClient', + help='Client to be used for sending request to engine'), +] + +CONF = cfg.CONF +CONF.register_opts(ENGINE_CLIENT_OPT, group='default') + +# Fetch scheduler defined in the configuration file and load it. +engine_client_info = CONF.default.engine_client +# May be this path can be specified in the configuration file. +engine_client_loc = 'dockyard.engine_client' +engine_client_info = (('%s.%s') % (engine_client_loc, engine_client_info)) +module_name, class_name = scheduler_info.rsplit(".", 1) +class_ = getattr(importlib.import_module(module_name), class_name) +engine_client = class_() + class URL(object): def __init__(self): - pass + self.engine_client = engine_client() @abc.abstractmethod def send(self, method, url, headers=None, post_params=None, @@ -43,13 +61,18 @@ def send(self, method, url, headers=None, post_params=None, :body: Request body. """ req_type = self._is_local_request() + # Pre processing needs to be done for dockyard feature before + # performing some actions if req_type: - preprocess + self.engine_client.pre_process(method=method, url=url, + body=body, headers=headers) else: headers = self._add_headers(headers) data = self.pool.urlopen(method, url, headers=headers, body=body).data - # post_process + # Post processing needs to be done for some of the dockyard operations + self.engine_client.post_process(method=method, url=url, + body=body, headers=headers) return data diff --git a/dockyard/engine/common/network/drivers/bridges/base.py b/dockyard/engine/common/network/drivers/bridges/base.py index b9753de..e737c33 100644 --- a/dockyard/engine/common/network/drivers/bridges/base.py +++ b/dockyard/engine/common/network/drivers/bridges/base.py @@ -240,8 +240,8 @@ def assign_ip(self, ifname, address, mask, broadcast=None, psid=None): self.addr.add(ifname, address, mask, broadcast, net_ns_fd=netns_name) except Exception as e: - msg = ("Unable to assign ip %s to %s interface in %s namespace." - "ERROR: %s" % (address, ifname, net_ns_fd, e)) + msg = ("Unable to assign ip %s to %s interface in psid namespace." + "ERROR: %s" % (address, ifname, psid, e)) raise UnableToAssignIP(msg) diff --git a/dockyard/engine/common/network/drivers/bridges/linux.py b/dockyard/engine/common/network/drivers/bridges/linux.py index dc9abbe..3868443 100644 --- a/dockyard/engine/common/network/drivers/bridges/linux.py +++ b/dockyard/engine/common/network/drivers/bridges/linux.py @@ -59,7 +59,6 @@ def create_link_pair(self, ifname=None, kind='veth', peer=None): except Exception as e: msg = ("Unable to create %s, %s interfaces of %s kind. ERROR: %s" % (ext_if, int_if, kind, e)) - return msg return {'ext_if': ext_if, 'int_if': int_if} diff --git a/dockyard/engine/common/network/manager.py b/dockyard/engine/common/network/manager.py index 984ab08..859c63f 100644 --- a/dockyard/engine/common/network/manager.py +++ b/dockyard/engine/common/network/manager.py @@ -33,7 +33,7 @@ def get_ifs(self, psid): ifs = dict() ifs[psid] = self.network.get_ifs(psid) - return json_dump(ifs) + return ifs def create(self, peer=None, ifname=None, kind='veth'): """This method creates interfaces in the namespace. @@ -47,7 +47,7 @@ def create(self, peer=None, ifname=None, kind='veth'): self.network.create_link_pair(ifname=ifname, peer=peer, kind=kind) - return json_dump(ifs_names) + return ifs_names def update(self, ifname, brname=None, psid=None, state=None): """This method is responsible for attaching network interface to @@ -74,7 +74,7 @@ def update(self, ifname, brname=None, psid=None, state=None): data["state"] = self.network.change_state(ifname=ifname, psid=psid, state=state) - return json_dump(data) + return data class IPManager(object): def __init__(self): @@ -92,7 +92,70 @@ def addr(self, ifname, address, mask, :returns: returns. """ ips = dict() - ips['IP'] = self.ip.addr(ifname=ifname, address=address, mask=mask, - psid=psid, broadcast=broadcast) + ips['IP'] = self.ip.addr(ifname=ifname, address=address, + mask=int(mask), psid=int(psid), + broadcast=broadcast) - return json_dump(ips) + return ips + +class RouteManager(object): + def __init__(self): + self.route = class_() + + def routes(self, ifname, gateway, psid, dst='default'): + """Adds route in the given namespace. + :params oif_name: interface name. + :params gateway: gateway to be added. + :params psid: process id of the docker to which gateway + has to added. + :params dst: destinations for the route. + + :returns: rotue information. + """ + routes = dict() + psid = int(psid) + routes["routes"] = self.route.add_routes(oif_name=ifname, dst=dst, + psid=psid, gateway=gateway) + + return routes + + +class DockyardNetworkManager(object): + """It handles all the dockyard networking specific settings. + """ + def __init__(self): + """Dockyard networking involve playing around with network interfaces + network routes and ip address. + """ + self.ip = IPManager() + self.route = RouteManager() + self.if_ = IFManager() + + def add_ip(self, ip, gateway, mask, psid): + """This method uses RouteManages, IPManager, IFManager to perform + required actions. + + :params ip: IP Address to be assigned to the container. + :params gateway: Gateway to be assigned to the container. + :params mask: netmask for the network address. + :params psid: process id for the docker container to which + this IP has to be assigned. + + :returns: returns information set to the container. + """ + # Create network inerfaces. + ifs = self.if_.create()["interface_names"] + + # Move network interfaces to the namespace + psid = int(psid) + self.if_.update(ifname=ifs["int_if"], psid=psid, state="up") + + # Assign IP address to the container + self.ip.addr(ifname=ifs["int_if"], psid=psid, address=ip, + mask=int(mask)) + + # Create routes for the newly added interface + self.route.routes(ifname=ifs["int_if"], psid=psid, gateway=gateway, + dst='default') + + return "Gathered Information." diff --git a/dockyard/engine/controllers/network.py b/dockyard/engine/controllers/network.py index 78eb143..e1bbc23 100644 --- a/dockyard/engine/controllers/network.py +++ b/dockyard/engine/controllers/network.py @@ -2,12 +2,19 @@ from pecan.rest import RestController from dockyard.engine.common.utils import str_to_dict -from dockyard.engine.common.network.manager import IFManager, IPManager +from dockyard.engine.common.network.manager import ( + IFManager, + IPManager, + RouteManager, + DockyardNetworkManager) + +from dockyard.engine.controllers.utils import do_not_expose class InterfaceController(RestController): def __init__(self): self.network = IFManager() + @do_not_expose @expose() def post(self): """This method is responsible for creating network interfaces. @@ -15,21 +22,25 @@ def post(self): kwargs = str_to_dict(request.body) return self.network.create(**kwargs) + @do_not_expose @expose() def delete(self): - pass + abort(404) + @do_not_expose @expose() def put(self): kwargs = str_to_dict(request.body) return self.network.update(**kwargs) + @do_not_expose @expose() def detach(self): - pass + abort(404) # created this for testing purpose only -# @expose() + @do_not_expose + @expose() def get_one(self, psid): """This method returns networks interfaces in a container. """ @@ -38,41 +49,61 @@ def get_one(self, psid): class RoutesController(RestController): def __init__(self): - pass + self.routes = RouteManager() + @do_not_expose @expose() - def post(self): - pass - def delete(self): - pass + abort(404) - def put(self): - pass + @do_not_expose + @expose() + def post(self): + kwargs = str_to_dict(request.body) + return self.routes.routes(**kwargs) + @do_not_expose + @expose() def get_one(self): - pass + abort(404) class IPController(RestController): def __init__(self): self.ip = IPManager() + @do_not_expose @expose() - def put(self): + def post(self): kwargs = str_to_dict(request.body) return self.ip.addr(**kwargs) + @do_not_expose @expose() def delete(self): - pass + abort(404) + @do_not_expose @expose() def get_one(self, psid=None): """This method returns IPs allocated to a container. """ - pass - + abort(404) + + +class DockyardNetworkController(RestController): + """This class expose dockyard network API. + """ + def __init__(self): + self.network = DockyardNetworkManager() + + @expose() + def post(self): + """It handles floating IP assignment for dockyard containers. + """ + kwargs = str_to_dict(request.body) + return self.network.add_ip(**kwargs) + class DockyardEngineController(RestController): """Controller for the Engine. @@ -84,11 +115,15 @@ class DockyardEngineController(RestController): """Controller for IPs. """ - ip = IPController() + # ip = IPController() """Controller for routes. """ - routes = RoutesController() + # routes = RoutesController() + """This controller is for providing dockyard specific networking. + """ + network = DockyardNetworkController() + def __init__(self): pass diff --git a/dockyard/engine/controllers/utils.py b/dockyard/engine/controllers/utils.py new file mode 100644 index 0000000..333316b --- /dev/null +++ b/dockyard/engine/controllers/utils.py @@ -0,0 +1,6 @@ +from pecan import expose + +def do_not_expose(method): + def dont_expose(self, *args, **kwargs): + return self.method(*args, **kwargs) + return dont_expose diff --git a/dockyard/engine_client/__init__.py b/dockyard/engine_client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine_client/api_server/api_server.py b/dockyard/engine_client/api_server/api_server.py new file mode 100644 index 0000000..ffd625e --- /dev/null +++ b/dockyard/engine_client/api_server/api_server.py @@ -0,0 +1,27 @@ +# This client will send request to api server, it sends request to API server + +from dockyard.engine_client.clients.base import EngineClient +from dockyard.engine_client.clients.containers import PreProcessor +from dockyard.engine_client.clients.containers import PostProcessor + + +class APIServerEngineClient(EngineClient): + """This method sends request to engine client through api server. + API server is made to listen for dockyard related CLI. It receives + request and then perform operation. + """ + def __init__(self): + self.pre_process = PreProcessor() + self.post_process = PostProcessor() + + def _get_module(self, url): + return url.rsplit(':', 1)[-1].split('/')[1] + + def pre_process(self, url, **kwargs): + module = self._get_module(url, **kwargs) + return getattr(self.pre_process, module)(url, **kwargs) + + + def post_process(self, url, **kwargs): + module = self._get_module(url, **kwargs) + return getattr(self.post_process, module)(url, **kwargs) diff --git a/dockyard/engine_client/base.py b/dockyard/engine_client/base.py new file mode 100644 index 0000000..c0797b3 --- /dev/null +++ b/dockyard/engine_client/base.py @@ -0,0 +1,20 @@ +# This module acts as a base class for the other clients. All the clients must +# inherit this class +import abc + +class EngineClient(object): + def __init__(self): + pass + + @abc.abstractmethod + def pre_process(self, **kwargs): + """This method is responsible for sending requests related to dockyard + before launching container. + """ + + @abc.abstractmethod + def post_process(self, **kwargs): + """This method is reponsible for sending request related to dockyard + after container has been created. + """ + pass diff --git a/test/resize b/test/resize old mode 100755 new mode 100644 diff --git a/test/test_engine.sh b/test/test_engine.sh new file mode 100644 index 0000000..9a2a13a --- /dev/null +++ b/test/test_engine.sh @@ -0,0 +1,10 @@ +count=2 +psid=$1 + +while [ $count -lt 254 ] +do + ip="10.0.9.$count" + abc=`curl http://10.0.9.1:5869/dockyard/network -XPOST -d "{'psid': '$psid', 'ip': '$ip', 'gateway': '10.0.9.1', 'mask': '24'}"` + count=`expr $count + 1` + echo "adding ..... $ip" +done diff --git a/test/update b/test/update old mode 100755 new mode 100644 From 6a2260c77d87f3774fa902ba8eb1c4d74470e209 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Mon, 30 May 2016 22:36:20 +0530 Subject: [PATCH 45/62] Added controllers for dockyard engine. --- dockyard/common/base.py | 6 +- dockyard/common/cluster/cluster.py | 2 +- dockyard/common/network/network.py | 22 ++- dockyard/common/url.py | 26 +++- dockyard/controllers/v1/network.py | 12 +- dockyard/engine/controllers/dockyard.py | 24 ++++ dockyard/engine/controllers/floatingip.py | 14 ++ dockyard/engine/controllers/interface.py | 43 ++++++ dockyard/engine/controllers/ip.py | 29 ++++ dockyard/engine/controllers/network.py | 127 +----------------- dockyard/engine/controllers/routes.py | 28 ++++ dockyard/engine_client/api_server/__init__.py | 0 .../engine_client/api_server/api_server.py | 7 +- .../api_server/container/__init__.py | 0 .../api_server/container/container.py | 7 + 15 files changed, 210 insertions(+), 137 deletions(-) create mode 100644 dockyard/engine/controllers/dockyard.py create mode 100644 dockyard/engine/controllers/floatingip.py create mode 100644 dockyard/engine/controllers/interface.py create mode 100644 dockyard/engine/controllers/ip.py create mode 100644 dockyard/engine/controllers/routes.py create mode 100644 dockyard/engine_client/api_server/__init__.py create mode 100644 dockyard/engine_client/api_server/container/__init__.py create mode 100644 dockyard/engine_client/api_server/container/container.py diff --git a/dockyard/common/base.py b/dockyard/common/base.py index d01b041..ac44770 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -1,11 +1,13 @@ import abc +import importlib +from oslo_config import cfg from pecan import request from urllib3 import PoolManager ENGINE_CLIENT_OPT = [ cfg.StrOpt('engine_client', default='api_server.api_server.APIServerEngineClient', - help='Client to be used for sending request to engine'), + help='Client to be used for sending request to engine') ] CONF = cfg.CONF @@ -16,7 +18,7 @@ # May be this path can be specified in the configuration file. engine_client_loc = 'dockyard.engine_client' engine_client_info = (('%s.%s') % (engine_client_loc, engine_client_info)) -module_name, class_name = scheduler_info.rsplit(".", 1) +module_name, class_name = engine_client_info.rsplit(".", 1) class_ = getattr(importlib.import_module(module_name), class_name) engine_client = class_() diff --git a/dockyard/common/cluster/cluster.py b/dockyard/common/cluster/cluster.py index fbb51f9..5f72747 100644 --- a/dockyard/common/cluster/cluster.py +++ b/dockyard/common/cluster/cluster.py @@ -13,5 +13,5 @@ def unregister(self, cluster_id, host_ip): pass def get_hosts(self): - LOG.debug("Registered hosts: %s" cfg.CONF.membership.hosts) + LOG.debug("Registered hosts: %s" % cfg.CONF.membership.hosts) return cfg.CONF.membership.hosts diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py index 36c9f0a..1d71fbb 100644 --- a/dockyard/common/network/network.py +++ b/dockyard/common/network/network.py @@ -4,7 +4,7 @@ from dockyard.common import url, utils -class Network(object): +class DockerNetwork(object): base_url = '/networks' def __init__(self): @@ -32,3 +32,23 @@ def create(self, data): def delete(self, id_): url_ = self.url.make_url(id_=id_) return utils.dispatch_delete_request(url=url_) + + +class DockyardNetwork(object): + dockyard_base_url = '/dockyard' + base_url = '/networks' + + def __init__(self): + self.url = url.URL(self.base_url) + + def attach_floatingip(self, id_, **kwargs): + """This method attaches floating ip to the containers. + """ + url_ = self.url.make_url(id_=id_, url_='/floatingip') + return utils.dispatch_post_request(url=url_) + + +class Network(DockerNetwork, DockyardNetwork): + def __init__(self): + super(DockerNetwork, self).__init__(self) + super(DockyardNetwork, self).__init__(self) diff --git a/dockyard/common/url.py b/dockyard/common/url.py index adfa73e..f72c4cd 100644 --- a/dockyard/common/url.py +++ b/dockyard/common/url.py @@ -2,11 +2,14 @@ class URL(object): def __init__(self, base_url): self.base_url = base_url - def make_url(self, url_=None, id_=None): - """It takes id, url as input and makes complete url, first part - of url is fetched from object itself. + def _add_url_id(self, url=None, sub_url=None, id_=None): + """This method takes base_url and url and ids and combine them + to make a base url. + + It creates a URL of the format /command/id. """ - url = self.base_url + if not url: + url = "" if id_: url = (("%s/%s") % (url, id_)) @@ -14,4 +17,19 @@ def make_url(self, url_=None, id_=None): if url_: url = (("%s/%s") % (url, url_)) + return url + + def make_url(self, url_=None, id_=None): + """It takes id, url as input and makes complete url, first part + of url is fetched from object itself. + """ + url = self._add_url_id(self.base_url, sub_url=url_, id_=id_) return url + + def make_dockyard_url(self, url_=None, id_=None): + """This method makes url for the API specifics to dockyard only. + This method takes an extra argument of creating dockayrd_base + which is corresponding to the dockayrd abse URL. + """ + url = (("%s/%s") % (self.dockyard_base_url, self.base_url)) + return self._add_url_id(self, url=url, sub_url=url_, id_=id_) diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py index 1fab92c..f160b80 100644 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -21,7 +21,7 @@ def connect(self): @connect.when(method='POST') def connect_(self, _id, **kwargs): - return self.network.connect(_id, kwargs) + return self.network.connect(_id, **kwargs) @expose(generic=True) def disconnect(self): @@ -29,7 +29,15 @@ def disconnect(self): @disconnect.when(method='POST') def disconnect_(self, _id, **kwargs): - return self.network.disconnect(_id, kwargs) + return self.network.disconnect(_id, **kwargs) + + @expose(generic=True) + def floatingip(self): + abort(404) + + @disconnect.when(method='POST') + def floating_(self, _id, **kwargs): + return self.network.attach_floatingip(_id, **kwargs) class NetworkController(object): diff --git a/dockyard/engine/controllers/dockyard.py b/dockyard/engine/controllers/dockyard.py new file mode 100644 index 0000000..ea3f6b7 --- /dev/null +++ b/dockyard/engine/controllers/dockyard.py @@ -0,0 +1,24 @@ +from dockyard.engine.controllers.network import DockyardNetworkController + +class DockyardEngineController(RestController): + """Controller for the Engine. + """ + + """This is the controller for the interface. + """ + interface = InterfaceController() + + """Controller for IPs. + """ + # ip = IPController() + + """Controller for routes. + """ + # routes = RoutesController() + + """This controller is for providing dockyard specific networking. + """ + network = DockyardNetworkController() + + def __init__(self): + pass diff --git a/dockyard/engine/controllers/floatingip.py b/dockyard/engine/controllers/floatingip.py new file mode 100644 index 0000000..e1b7299 --- /dev/null +++ b/dockyard/engine/controllers/floatingip.py @@ -0,0 +1,14 @@ +from dockyard.engine.common.network.manager import DockyardNetworkManager + +class FloatingIPController(object): + def __init__(self): + self.network = DockyardNetworkManager() + + @expose() + def post(self): + """It handles floating IP assignment for dockyard containers. + """ + kwargs = str_to_dict(request.body) + return self.network.add_ip(**kwargs) + + diff --git a/dockyard/engine/controllers/interface.py b/dockyard/engine/controllers/interface.py new file mode 100644 index 0000000..fd591d5 --- /dev/null +++ b/dockyard/engine/controllers/interface.py @@ -0,0 +1,43 @@ +from pecan import expose, abort, request +from pecan.rest import RestController +from dockyard.engine.common.utils import str_to_dict +from dockyard.engine.common.network.manager import IFManager +from dockyard.engine.controllers.utils import do_not_expose + +class InterfaceController(RestController): + def __init__(self): + self.network = IFManager() + + @do_not_expose + @expose() + def post(self): + """This method is responsible for creating network interfaces. + """ + kwargs = str_to_dict(request.body) + return self.network.create(**kwargs) + + @do_not_expose + @expose() + def delete(self): + abort(404) + + @do_not_expose + @expose() + def put(self): + kwargs = str_to_dict(request.body) + return self.network.update(**kwargs) + + @do_not_expose + @expose() + def detach(self): + abort(404) + + # created this for testing purpose only + @do_not_expose + @expose() + def get_one(self, psid): + """This method returns networks interfaces in a container. + """ + psid = int(psid) + return self.network.get_ifs(psid=psid) + diff --git a/dockyard/engine/controllers/ip.py b/dockyard/engine/controllers/ip.py new file mode 100644 index 0000000..fbd442a --- /dev/null +++ b/dockyard/engine/controllers/ip.py @@ -0,0 +1,29 @@ +from pecan import expose, abort, request +from pecan.rest import RestController +from dockyard.engine.common.utils import str_to_dict +from dockyard.engine.common.network.manager import IPManager +from dockyard.engine.controllers.utils import do_not_expose + +class IPController(RestController): + def __init__(self): + self.ip = IPManager() + + @do_not_expose + @expose() + def post(self): + kwargs = str_to_dict(request.body) + return self.ip.addr(**kwargs) + + @do_not_expose + @expose() + def delete(self): + abort(404) + + @do_not_expose + @expose() + def get_one(self, psid=None): + """This method returns IPs allocated to a container. + """ + abort(404) + + diff --git a/dockyard/engine/controllers/network.py b/dockyard/engine/controllers/network.py index e1bbc23..449da5f 100644 --- a/dockyard/engine/controllers/network.py +++ b/dockyard/engine/controllers/network.py @@ -1,129 +1,8 @@ -from pecan import expose, abort, request -from pecan.rest import RestController - -from dockyard.engine.common.utils import str_to_dict -from dockyard.engine.common.network.manager import ( - IFManager, - IPManager, - RouteManager, - DockyardNetworkManager) - -from dockyard.engine.controllers.utils import do_not_expose - -class InterfaceController(RestController): - def __init__(self): - self.network = IFManager() - - @do_not_expose - @expose() - def post(self): - """This method is responsible for creating network interfaces. - """ - kwargs = str_to_dict(request.body) - return self.network.create(**kwargs) - - @do_not_expose - @expose() - def delete(self): - abort(404) - - @do_not_expose - @expose() - def put(self): - kwargs = str_to_dict(request.body) - return self.network.update(**kwargs) - - @do_not_expose - @expose() - def detach(self): - abort(404) - - # created this for testing purpose only - @do_not_expose - @expose() - def get_one(self, psid): - """This method returns networks interfaces in a container. - """ - psid = int(psid) - return self.network.get_ifs(psid=psid) - -class RoutesController(RestController): - def __init__(self): - self.routes = RouteManager() - - @do_not_expose - @expose() - def delete(self): - abort(404) - - @do_not_expose - @expose() - def post(self): - kwargs = str_to_dict(request.body) - return self.routes.routes(**kwargs) - - @do_not_expose - @expose() - def get_one(self): - abort(404) - - -class IPController(RestController): - def __init__(self): - self.ip = IPManager() - - @do_not_expose - @expose() - def post(self): - kwargs = str_to_dict(request.body) - return self.ip.addr(**kwargs) - - @do_not_expose - @expose() - def delete(self): - abort(404) - - @do_not_expose - @expose() - def get_one(self, psid=None): - """This method returns IPs allocated to a container. - """ - abort(404) - +from dockyard.engine.controllers.floatingip import FloatingIPController +from dockyard.engine.controllers.interface import InterfaceController class DockyardNetworkController(RestController): """This class expose dockyard network API. """ - def __init__(self): - self.network = DockyardNetworkManager() - - @expose() - def post(self): - """It handles floating IP assignment for dockyard containers. - """ - kwargs = str_to_dict(request.body) - return self.network.add_ip(**kwargs) - - -class DockyardEngineController(RestController): - """Controller for the Engine. - """ - - """This is the controller for the interface. - """ + floatingip = FlotingIPController() interface = InterfaceController() - - """Controller for IPs. - """ - # ip = IPController() - - """Controller for routes. - """ - # routes = RoutesController() - - """This controller is for providing dockyard specific networking. - """ - network = DockyardNetworkController() - - def __init__(self): - pass diff --git a/dockyard/engine/controllers/routes.py b/dockyard/engine/controllers/routes.py new file mode 100644 index 0000000..b9d9ea2 --- /dev/null +++ b/dockyard/engine/controllers/routes.py @@ -0,0 +1,28 @@ +from pecan import expose, abort, request +from pecan.rest import RestController + +from dockyard.engine.common.utils import str_to_dict +from dockyard.engine.common.network.manager import RouteManager +from dockyard.engine.controllers.utils import do_not_expose + +class RoutesController(RestController): + def __init__(self): + self.routes = RouteManager() + + @do_not_expose + @expose() + def delete(self): + abort(404) + + @do_not_expose + @expose() + def post(self): + kwargs = str_to_dict(request.body) + return self.routes.routes(**kwargs) + + @do_not_expose + @expose() + def get_one(self): + abort(404) + + diff --git a/dockyard/engine_client/api_server/__init__.py b/dockyard/engine_client/api_server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine_client/api_server/api_server.py b/dockyard/engine_client/api_server/api_server.py index ffd625e..8fbd9f6 100644 --- a/dockyard/engine_client/api_server/api_server.py +++ b/dockyard/engine_client/api_server/api_server.py @@ -1,8 +1,9 @@ # This client will send request to api server, it sends request to API server -from dockyard.engine_client.clients.base import EngineClient -from dockyard.engine_client.clients.containers import PreProcessor -from dockyard.engine_client.clients.containers import PostProcessor +from dockyard.engine_client.base import EngineClient +from dockyard.engine_client.api_server.container.container import ( + PreProcessor, + PostProcessor) class APIServerEngineClient(EngineClient): diff --git a/dockyard/engine_client/api_server/container/__init__.py b/dockyard/engine_client/api_server/container/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine_client/api_server/container/container.py b/dockyard/engine_client/api_server/container/container.py new file mode 100644 index 0000000..2a3918f --- /dev/null +++ b/dockyard/engine_client/api_server/container/container.py @@ -0,0 +1,7 @@ +class PreProcessor(object): + def __ini__(self): + pass + +class PostProcessor(object): + def __init__(self): + pass From 1e8a311f32dfbba95340301391a1d62d9923b926 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Tue, 31 May 2016 22:45:53 +0530 Subject: [PATCH 46/62] updated for dockyard engine APIs --- dockyard/common/base.py | 6 ++-- dockyard/common/network/network.py | 22 ++++++++------ dockyard/common/url.py | 30 +++++++++++++++---- dockyard/common/utils.py | 29 ++++++++++++++---- dockyard/controllers/v1/__init__.py | 3 +- dockyard/controllers/v1/network.py | 9 +++--- .../common/network/drivers/bridges/base.py | 1 + dockyard/engine/common/network/manager.py | 1 + .../{dockyard.py => dockyard_controllers.py} | 4 ++- dockyard/engine/controllers/floatingip.py | 8 +++-- dockyard/engine/controllers/network.py | 6 +++- 11 files changed, 87 insertions(+), 32 deletions(-) rename dockyard/engine/controllers/{dockyard.py => dockyard_controllers.py} (77%) diff --git a/dockyard/common/base.py b/dockyard/common/base.py index ac44770..477855a 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -70,11 +70,11 @@ def send(self, method, url, headers=None, post_params=None, body=body, headers=headers) else: headers = self._add_headers(headers) - + print url data = self.pool.urlopen(method, url, headers=headers, body=body).data # Post processing needs to be done for some of the dockyard operations - self.engine_client.post_process(method=method, url=url, - body=body, headers=headers) + #self.engine_client.post_process(method=method, url=url, + # body=body, headers=headers) return data diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py index 1d71fbb..6c1d2c9 100644 --- a/dockyard/common/network/network.py +++ b/dockyard/common/network/network.py @@ -8,7 +8,7 @@ class DockerNetwork(object): base_url = '/networks' def __init__(self): - self.url = url.URL(self.base_url) + pass def list(self, name_or_id=None): url_ = self.url.make_url(id_=name_or_id) @@ -36,19 +36,23 @@ def delete(self, id_): class DockyardNetwork(object): dockyard_base_url = '/dockyard' - base_url = '/networks' def __init__(self): - self.url = url.URL(self.base_url) + pass + + def _get_localhost(self): + return utils.get_localhost() - def attach_floatingip(self, id_, **kwargs): + def attach_floatingip(self, id_, data): """This method attaches floating ip to the containers. """ - url_ = self.url.make_url(id_=id_, url_='/floatingip') - return utils.dispatch_post_request(url=url_) + url_ = self.url.make_dockyard_url(id_=id_, url_='floatingip') + return utils.dispatch_post_request(url=url_, body=body, + host=self._get_localhost()) -class Network(DockerNetwork, DockyardNetwork): +class Network(DockyardNetwork, DockerNetwork): def __init__(self): - super(DockerNetwork, self).__init__(self) - super(DockyardNetwork, self).__init__(self) + super(DockerNetwork, self).__init__() + super(DockyardNetwork, self).__init__() + self.url = url.URL(self.base_url, self.dockyard_base_url) diff --git a/dockyard/common/url.py b/dockyard/common/url.py index f72c4cd..be3a504 100644 --- a/dockyard/common/url.py +++ b/dockyard/common/url.py @@ -1,6 +1,8 @@ class URL(object): - def __init__(self, base_url): + def __init__(self, base_url, dockyard_base_url=None): self.base_url = base_url + if dockyard_base_url: + self.dockyard_base_url = dockyard_base_url def _add_url_id(self, url=None, sub_url=None, id_=None): """This method takes base_url and url and ids and combine them @@ -14,8 +16,8 @@ def _add_url_id(self, url=None, sub_url=None, id_=None): if id_: url = (("%s/%s") % (url, id_)) - if url_: - url = (("%s/%s") % (url, url_)) + if sub_url: + url = (("%s/%s") % (url, sub_url)) return url @@ -25,11 +27,29 @@ def make_url(self, url_=None, id_=None): """ url = self._add_url_id(self.base_url, sub_url=url_, id_=id_) return url + + def _add_dockyard_url(self, url=None, sub_url=None, id_=None): + """This method takes base_url and url and ids and combine them + to make a base url. + + It creates a URL of the format /command/id. + """ + if not url: + url = "" + + if sub_url: + url = (("%s/%s") % (url, sub_url)) + + if id_: + url = (("%s/%s") % (url, id_)) + + return url + def make_dockyard_url(self, url_=None, id_=None): """This method makes url for the API specifics to dockyard only. This method takes an extra argument of creating dockayrd_base which is corresponding to the dockayrd abse URL. """ - url = (("%s/%s") % (self.dockyard_base_url, self.base_url)) - return self._add_url_id(self, url=url, sub_url=url_, id_=id_) + url = (("%s%s") % (self.dockyard_base_url, self.base_url)) + return self._add_dockyard_url(url=url, sub_url=url_, id_=id_) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index 81d73f0..debee3d 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -75,9 +75,22 @@ def get_host(): return host +def get_localhost(): + d = dict() + + if CONF.default.host == '0.0.0.0': + d['host'] = '127.0.0.1' + else: + d['host'] = CONF.default.host + + d['port'] = CONF.default.port + return d + + +def get_link(url, host=None, protocol='http'): + if not host: + host = get_host() -def get_link(url, protocol='http'): - host = get_host() return link.make_url(host=host['host'], port=host['port'], url=url) def prepare_logging(argv=None): @@ -87,6 +100,7 @@ def prepare_logging(argv=None): """ if argv is None: argv = [] + logging.register_options(CONF) CONF(argv[1:], project='dockyard') logging.setup(CONF, 'dockyard') @@ -101,8 +115,10 @@ def dispatch_get_request(url, headers=None, protocol='http', query_params=None): return request.send(method='GET', url=ln) -def dispatch_post_request(url, protocol='http', body=None, query_params=None): - ln = get_link(url, protocol) +def dispatch_post_request(url, host=None, protocol='http', + body=None, query_params=None): + + ln = get_link(host=host, url=url, protocol=protocol) if query_params: query = link.make_query_url(query_params) @@ -127,10 +143,11 @@ def dispatch_delete_request(url, headers = None, protocol='http', return request.send(method='DELETE', url=ln) -def dispatch_post_req(url, headers=None, body=None, post_params=None): +def dispatch_post_req(url, headers=None, body=None, + post_params=None, host=None): if not headers: headers = {'Content-Type': 'application/json'} - + return request.send(method='POST', url=url, headers=headers, body=body) diff --git a/dockyard/controllers/v1/__init__.py b/dockyard/controllers/v1/__init__.py index 98cef76..5738274 100644 --- a/dockyard/controllers/v1/__init__.py +++ b/dockyard/controllers/v1/__init__.py @@ -8,7 +8,8 @@ from dockyard.controllers.v1 import network from dockyard.controllers.v1 import volume from dockyard.controllers.v1 import cluster -from dockyard.engine.controllers.network import DockyardEngineController +from dockyard.engine.controllers.dockyard_controllers import ( + DockyardEngineController) SERVER_OPT = [ diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py index f160b80..3927647 100644 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -35,9 +35,9 @@ def disconnect_(self, _id, **kwargs): def floatingip(self): abort(404) - @disconnect.when(method='POST') - def floating_(self, _id, **kwargs): - return self.network.attach_floatingip(_id, **kwargs) + @floatingip.when(method='POST') + def floatingip_(self, _id, **kwargs): + return self.network.attach_floatingip(_id, kwargs) class NetworkController(object): @@ -54,6 +54,7 @@ def _create(self, **kwargs): @expose() def _lookup(self, id_name_op=None, op=None): + new_url = [] if op: new_url.append(op) @@ -66,5 +67,5 @@ def _lookup(self, id_name_op=None, op=None): new_url = tuple(new_url) else: new_url = tuple(['']) - + return Network(), new_url diff --git a/dockyard/engine/common/network/drivers/bridges/base.py b/dockyard/engine/common/network/drivers/bridges/base.py index e48f009..29f6926 100644 --- a/dockyard/engine/common/network/drivers/bridges/base.py +++ b/dockyard/engine/common/network/drivers/bridges/base.py @@ -129,6 +129,7 @@ def __init__(self): def create(self, ifname, peer, kind='veth', net_ns_fd=None): """Create link. """ + print ifname, peer, kind, net_ns_fd ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) ipdb.create(ifname=ifname, kind=kind, peer=peer) self.ipdb_manager.close_ipdb(ipdb) diff --git a/dockyard/engine/common/network/manager.py b/dockyard/engine/common/network/manager.py index 859c63f..5804d37 100644 --- a/dockyard/engine/common/network/manager.py +++ b/dockyard/engine/common/network/manager.py @@ -148,6 +148,7 @@ def add_ip(self, ip, gateway, mask, psid): # Move network interfaces to the namespace psid = int(psid) + print ifs self.if_.update(ifname=ifs["int_if"], psid=psid, state="up") # Assign IP address to the container diff --git a/dockyard/engine/controllers/dockyard.py b/dockyard/engine/controllers/dockyard_controllers.py similarity index 77% rename from dockyard/engine/controllers/dockyard.py rename to dockyard/engine/controllers/dockyard_controllers.py index ea3f6b7..aca2d59 100644 --- a/dockyard/engine/controllers/dockyard.py +++ b/dockyard/engine/controllers/dockyard_controllers.py @@ -1,4 +1,6 @@ +from pecan.rest import RestController from dockyard.engine.controllers.network import DockyardNetworkController +from dockyard.engine.controllers.interface import InterfaceController class DockyardEngineController(RestController): """Controller for the Engine. @@ -18,7 +20,7 @@ class DockyardEngineController(RestController): """This controller is for providing dockyard specific networking. """ - network = DockyardNetworkController() + networks = DockyardNetworkController() def __init__(self): pass diff --git a/dockyard/engine/controllers/floatingip.py b/dockyard/engine/controllers/floatingip.py index e1b7299..9a93eaf 100644 --- a/dockyard/engine/controllers/floatingip.py +++ b/dockyard/engine/controllers/floatingip.py @@ -1,11 +1,15 @@ +from pecan import expose, request +from pecan.rest import RestController + +from dockyard.engine.common.utils import str_to_dict from dockyard.engine.common.network.manager import DockyardNetworkManager -class FloatingIPController(object): +class FloatingIPController(RestController): def __init__(self): self.network = DockyardNetworkManager() @expose() - def post(self): + def post(self, id_): """It handles floating IP assignment for dockyard containers. """ kwargs = str_to_dict(request.body) diff --git a/dockyard/engine/controllers/network.py b/dockyard/engine/controllers/network.py index 449da5f..1fe7a5e 100644 --- a/dockyard/engine/controllers/network.py +++ b/dockyard/engine/controllers/network.py @@ -1,8 +1,12 @@ +from pecan.rest import RestController from dockyard.engine.controllers.floatingip import FloatingIPController from dockyard.engine.controllers.interface import InterfaceController class DockyardNetworkController(RestController): """This class expose dockyard network API. """ - floatingip = FlotingIPController() + floatingip = FloatingIPController() interface = InterfaceController() + + def __init__(self): + pass From adb61a3376e5885c92bfe1525c1736f9d9e84f0c Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Wed, 1 Jun 2016 22:53:46 +0530 Subject: [PATCH 47/62] Corrections for attaching floating ip API --- dockyard/common/base.py | 3 ++- dockyard/common/network/network.py | 1 + .../common/network/drivers/bridges/__init__.py | 9 +++++++++ .../common/network/drivers/bridges/base.py | 1 - .../common/network/drivers/bridges/linux.py | 4 ++-- dockyard/engine/common/network/manager.py | 18 ++++++++++++++++-- dockyard/engine/controllers/__init__.py | 10 ++++++++-- dockyard/engine/controllers/floatingip.py | 2 -- etc/dockyard/dockyard.conf | 9 +++++++++ test/floatingip.py | 1 + 10 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 test/floatingip.py diff --git a/dockyard/common/base.py b/dockyard/common/base.py index 477855a..1e9ca2d 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -65,12 +65,13 @@ def send(self, method, url, headers=None, post_params=None, req_type = self._is_local_request() # Pre processing needs to be done for dockyard feature before # performing some actions + print method, url, headers, body if req_type: self.engine_client.pre_process(method=method, url=url, body=body, headers=headers) else: headers = self._add_headers(headers) - print url + data = self.pool.urlopen(method, url, headers=headers, body=body).data # Post processing needs to be done for some of the dockyard operations #self.engine_client.post_process(method=method, url=url, diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py index 6c1d2c9..4fe66f7 100644 --- a/dockyard/common/network/network.py +++ b/dockyard/common/network/network.py @@ -47,6 +47,7 @@ def attach_floatingip(self, id_, data): """This method attaches floating ip to the containers. """ url_ = self.url.make_dockyard_url(id_=id_, url_='floatingip') + body = request.body return utils.dispatch_post_request(url=url_, body=body, host=self._get_localhost()) diff --git a/dockyard/engine/common/network/drivers/bridges/__init__.py b/dockyard/engine/common/network/drivers/bridges/__init__.py index e69de29..113373f 100644 --- a/dockyard/engine/common/network/drivers/bridges/__init__.py +++ b/dockyard/engine/common/network/drivers/bridges/__init__.py @@ -0,0 +1,9 @@ +from oslo_config import cfg + +HOST_SERVICE_OPT = [ + cfg.StrOpt('bridge', + default='br100', + help='Bridge for the external connectivity network'), +] + +cfg.CONF.register_opts(HOST_SERVICE_OPT, 'network') diff --git a/dockyard/engine/common/network/drivers/bridges/base.py b/dockyard/engine/common/network/drivers/bridges/base.py index 29f6926..e48f009 100644 --- a/dockyard/engine/common/network/drivers/bridges/base.py +++ b/dockyard/engine/common/network/drivers/bridges/base.py @@ -129,7 +129,6 @@ def __init__(self): def create(self, ifname, peer, kind='veth', net_ns_fd=None): """Create link. """ - print ifname, peer, kind, net_ns_fd ipdb = self.ipdb_manager.open_ipdb(net_ns_fd) ipdb.create(ifname=ifname, kind=kind, peer=peer) self.ipdb_manager.close_ipdb(ipdb) diff --git a/dockyard/engine/common/network/drivers/bridges/linux.py b/dockyard/engine/common/network/drivers/bridges/linux.py index 92381f5..f53224c 100644 --- a/dockyard/engine/common/network/drivers/bridges/linux.py +++ b/dockyard/engine/common/network/drivers/bridges/linux.py @@ -72,10 +72,10 @@ def attach_port(self, ifname, br_name): """ try: self.link.attach_port(ifname=ifname, bridge=br_name) - LOG.info("Attached interface: %s to bridge: %s" % (ifname, bridge)) + LOG.info("Attached interface: %s to bridge: %s" % (ifname, br_name)) except Exception as e: msg = ("Unable to attach %s interface with %s bridge. ERROR: %s" - % (ifname, bridge, e)) + % (ifname, br_name, e)) LOG.exception(msg) raise UnableToAttachPort(msg) diff --git a/dockyard/engine/common/network/manager.py b/dockyard/engine/common/network/manager.py index 5804d37..6019085 100644 --- a/dockyard/engine/common/network/manager.py +++ b/dockyard/engine/common/network/manager.py @@ -6,7 +6,7 @@ CONF = cfg.CONF # Fetch scheduler defined in the configuration file and load it. -network_driver_info = CONF.default.network_driver +network_driver_info = CONF.network.network_driver # May be this path can be specified in the configuration file. network_driver_loc = 'dockyard.engine.common.network.drivers' network_driver_info = (('%s.%s') % (network_driver_loc, network_driver_info)) @@ -35,6 +35,13 @@ def get_ifs(self, psid): ifs[psid] = self.network.get_ifs(psid) return ifs + def attach_if(self, ifname, brname): + """This method attach interface to the bridge for external interface + :params ifname: interface name of the new container. + :params brname: bridge name to which container attach. + """ + self.network.attach_port(ifname=ifname, br_name=brname) + def create(self, peer=None, ifname=None, kind='veth'): """This method creates interfaces in the namespace. :params ifname: Interface name, if not defined random @@ -144,19 +151,26 @@ def add_ip(self, ip, gateway, mask, psid): :returns: returns information set to the container. """ # Create network inerfaces. + print "++++" ifs = self.if_.create()["interface_names"] + print "++++" # Move network interfaces to the namespace psid = int(psid) - print ifs self.if_.update(ifname=ifs["int_if"], psid=psid, state="up") + print "++++" # Assign IP address to the container self.ip.addr(ifname=ifs["int_if"], psid=psid, address=ip, mask=int(mask)) + print "++++" # Create routes for the newly added interface self.route.routes(ifname=ifs["int_if"], psid=psid, gateway=gateway, dst='default') + print "++++" + self.if_.attach_if(ifname=ifs["ext_if"], brname=CONF.network.bridge) + + print "++++" return "Gathered Information." diff --git a/dockyard/engine/controllers/__init__.py b/dockyard/engine/controllers/__init__.py index 0cdfd6b..692f9ea 100644 --- a/dockyard/engine/controllers/__init__.py +++ b/dockyard/engine/controllers/__init__.py @@ -1,6 +1,5 @@ from oslo_config import cfg - NETWORK_DRIVER_OPT = [ cfg.StrOpt('network_driver', default='bridges.linux.LinuxBridgeManager', @@ -8,4 +7,11 @@ ] CONF = cfg.CONF -CONF.register_opts(NETWORK_DRIVER_OPT, group='default') +opt_group = cfg.OptGroup(name='network', + title='Group for the network values of dockyard api') +CONF.register_group(opt_group) +CONF.register_opts(NETWORK_DRIVER_OPT, opt_group) + + +CONF = cfg.CONF +CONF.register_opts(NETWORK_DRIVER_OPT, group='network') diff --git a/dockyard/engine/controllers/floatingip.py b/dockyard/engine/controllers/floatingip.py index 9a93eaf..62fc392 100644 --- a/dockyard/engine/controllers/floatingip.py +++ b/dockyard/engine/controllers/floatingip.py @@ -14,5 +14,3 @@ def post(self, id_): """ kwargs = str_to_dict(request.body) return self.network.add_ip(**kwargs) - - diff --git a/etc/dockyard/dockyard.conf b/etc/dockyard/dockyard.conf index d8caf86..160df59 100644 --- a/etc/dockyard/dockyard.conf +++ b/etc/dockyard/dockyard.conf @@ -19,6 +19,13 @@ membership = consul_driver.Consul # This product can behave differently based on the role being given by it. agent = master +[network] + +# Network driver defines the way networking is being handled by dockyard. +network_driver = bridges.linux.LinuxBridgeManager + +bridge = br + [consul] service_name = 'dockyard' @@ -33,3 +40,5 @@ docker_port = 2375 # by other application also therefor dockyard can register under different # service name docker_name = 'docker' + + diff --git a/test/floatingip.py b/test/floatingip.py new file mode 100644 index 0000000..a204064 --- /dev/null +++ b/test/floatingip.py @@ -0,0 +1 @@ +curl http://127.0.0.1:5869/dockyard/networks/floatingip/7289 -XPOST -d '{"psid": "7289", "ip": "10.0.9.2", "mask": "24", "gateway": "10.0.9.1"}' From b1159c28114bc40468ad8fc29dd1398cdf71b08a Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sun, 5 Jun 2016 09:20:19 +0530 Subject: [PATCH 48/62] restructure of the code. --- dockyard/common/base.py | 2 +- dockyard/common/container/container.py | 6 +- dockyard/common/image/image.py | 2 +- dockyard/common/network/network.py | 12 +- dockyard/common/utils.py | 8 +- dockyard/common/volume/volume.py | 3 +- dockyard/controllers/v1/container.py | 164 +++++-------------------- dockyard/controllers/v1/image.py | 94 +++++++------- dockyard/controllers/v1/information.py | 9 +- dockyard/controllers/v1/network.py | 76 ++++-------- dockyard/controllers/v1/volume.py | 48 +++----- test/floatingip.py | 2 +- 12 files changed, 141 insertions(+), 285 deletions(-) diff --git a/dockyard/common/base.py b/dockyard/common/base.py index 1e9ca2d..1e6e01d 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -65,7 +65,7 @@ def send(self, method, url, headers=None, post_params=None, req_type = self._is_local_request() # Pre processing needs to be done for dockyard feature before # performing some actions - print method, url, headers, body + # print method, url, headers, body if req_type: self.engine_client.pre_process(method=method, url=url, body=body, headers=headers) diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index 76fc5da..b17b1c5 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -29,6 +29,7 @@ def changes(self, name_or_id=None): def resize(self, id_, **kwargs): url_ = self.url.make_url(url_='resize', id_=id_) + query = '' if kwargs: query = link.make_query_url(kwargs) @@ -65,6 +66,7 @@ def copy(self, id_): def logs(self, id_, query_string, **kwargs): url_ = self.url.make_url(url_='logs', id_=id_) + query = '' if kwargs: query = link.make_query_url(kwargs) @@ -84,7 +86,7 @@ def kill(self, id_, query_params=None): def stop(self, id_, query_params=None): url_ = self.url.make_url(url_='stop', id_=id_) - return utils.dispatch_post_request(url_, query_params=query_parmas) + return utils.dispatch_post_request(url_, query_params=query_params) def exe(self, _id): abort(404) @@ -96,6 +98,7 @@ def attach(self, id_, query_params=None): def rename(self, id_, **kwargs): url_ = self.url.make_url(url_='rename', id_=id_) + query = '' if kwargs: query = link.make_query_url(kwargs) @@ -120,6 +123,7 @@ def wait(self, id_): def delete(self, id_, **kwargs): url_ = self.url.make_url(id_=id_) + query = '' if kwargs: query = link.make_query_url(kwargs) diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py index 5b53ec1..523ddd3 100644 --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -15,7 +15,7 @@ def list(self, id_=None): def history(self, id_=None): url_ = self.url.make_url(url_='history', id_=id_) - return utils.dispatch_get_request(url=url_).data + return utils.dispatch_get_request(url=url_) def search(self, term=None): url_ = self.url.make_url(url_='search') diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py index 4fe66f7..ed5089d 100644 --- a/dockyard/common/network/network.py +++ b/dockyard/common/network/network.py @@ -14,19 +14,19 @@ def list(self, name_or_id=None): url_ = self.url.make_url(id_=name_or_id) return utils.dispatch_get_request(url=url_) - def connect(self, id_, data): - body = json.dumps(data) + def connect(self, id_, **kwargs): + body = request.body url_ = self.url.make_url(url_='connect', id_=id_) return utils.dispatch_post_request(url=url_, body=body) - def disconnect(self, id_, data): + def disconnect(self, id_, **kwargs): url_ = self.url.make_url(url_='disconnect', id_=id_) - body = json.dumps(data) + body = request.body return utils.dispatch_post_request(url=url_, body=body) - def create(self, data): + def create(self, **kwargs): url_ = self.url.make_url(url_='create') - body = json.dumps(data) + body = request.body return utils.dispatch_post_request(url=url_, body=body) def delete(self, id_): diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index debee3d..ff93e58 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -106,7 +106,7 @@ def prepare_logging(argv=None): logging.setup(CONF, 'dockyard') def dispatch_get_request(url, headers=None, protocol='http', query_params=None): - ln = get_link(url, protocol) + ln = get_link(url=url, protocol=protocol) if query_params: query = link.make_query_url(query_params) @@ -128,7 +128,7 @@ def dispatch_post_request(url, host=None, protocol='http', def dispatch_put_request(url, protocol='http', body=None, query_params=None): - ln = get_link(url, protocol) + ln = get_link(url=url, protocol=protocol) if query_params: query = link.make_query_url(query_params) @@ -139,7 +139,7 @@ def dispatch_put_request(url, protocol='http', body=None, query_params=None): def dispatch_delete_request(url, headers = None, protocol='http', query_params=None): - ln = get_link(url, protocol) + ln = get_link(url=url, protocol=protocol) return request.send(method='DELETE', url=ln) @@ -147,7 +147,7 @@ def dispatch_post_req(url, headers=None, body=None, post_params=None, host=None): if not headers: headers = {'Content-Type': 'application/json'} - + return request.send(method='POST', url=url, headers=headers, body=body) diff --git a/dockyard/common/volume/volume.py b/dockyard/common/volume/volume.py index fbb0a12..8efed72 100644 --- a/dockyard/common/volume/volume.py +++ b/dockyard/common/volume/volume.py @@ -19,5 +19,4 @@ def delete(self, name): def create(self, data): url_ = self.url.make_url(url_='create') - body = json.dumps(data) - return utils.dispatch_post_request(url, body=body) + return utils.dispatch_post_request(url=url_, body=data) diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py index 8ac91f9..8031041 100644 --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -1,211 +1,107 @@ from pecan import expose, abort, request +from pecan.rest import RestController from dockyard.common.container import container -class Container(object): +class ContainerController(RestController): def __init__(self): self.container = container.Container() - @expose(generic=True) - def index(self): - abort(404) + def _call_operation(self, name_or_id, operation, **kwargs): + try: + return getattr(self, operation)(name_or_id) + except AttributeError: + abort(404) - @index.when(method="DELETE") + @expose() + def get(self, name_or_id, operation, **kwargs): + return self._call_operation(name_or_id, operation, **kwargs) + + @expose() + def post(self, name_or_id, operation, **kwargs): + return self._call_operation(name_or_id, operation, **kwargs) + + @expose() + def put(self, name_or_id, operation, **kwargs): + return self._call_operation(name_or_id, operation, **kwargs) + + @expose() def delete(self, _id, **kwargs): return self.container.delete(_id, **kwargs) - @expose(generic=True) def resize(self, _id, **kwargs): - abort(404) - - @resize.when(method="POST") - def resize_POST(self, _id, **kwargs): return self.container.resize(_id, **kwargs) - @expose() def changes(self, _id): return self.container.changes(_id) - @expose() def export(self, _id): return self.container.export(_id) - @expose() def json(self, name_id=None): query_params = request.query_string return self.container.list(name_id, query_params=query_params) - @expose() def stats(self, _id): return self.container.stats(_id) - @expose(generic=True) def archive(self, _id): query_params = request.query_string return self.container.archive(_id, query_params) - @archive.when(method="PUT") def upload(self, _id, **kwargs): body = request.body query_params = request.query_string return self.container.upload(_id, body=body, query_params=query_params) - @expose(generic=True) - def copy(self, _id): - abort(404) - - @copy.when(method="POST") def copy_POST(self, _id): return self.container.copy(_id) - @expose() def logs(self, _id): query_params = request.query_string return self.container.logs(_id, query_params) - @expose(generic=True) - def start(self): - abort(404) - - @start.when(method="POST") - def start_POST(self, _id): + def start(self, _id): query_params = request.query_string return self.container.start(_id, query_params=query_params) - @expose(generic=True) - def kill(self): - abort(404) - - @kill.when(method="POST") - def kill_POST(self, _id): + def kill(self, _id): query_params = request.query_string return self.container.kill(_id, query_params=query_params) - @expose(generic=True) - def restart(self): - abort(404) - - @restart.when(method="POST") - def restart_POST(self, _id): + def restart(self, _id): return self.container.restart(_id) - @expose(generic=True) - def stop(self): - abort(404) - - @stop.when(method="POST") - def stop_POST(self, _id): + def stop(self, _id): query_params = request.query_string return self.container.stop(_id, query_params=query_params) - @expose(generic=True, route='exec') - def exe(self): - abort(404) - - @exe.when(method="POST") - def exec_POST(self, _id): + def exec_(self, _id): return self.container.exe(_id) - @expose(generic=True) - def attach(self): - abort(404) - - @attach.when(method="POST") - def attach_POST(self, _id): + def attach(self, _id): query_params = request.query_string return self.container.attach(_id, query_params=query_params) - @expose(generic=True) - def rename(self): - abort(404) - - @rename.when(method="POST") - def rename_POST(self, _id, **kwargs): + def rename(self, _id, **kwargs): return self.container.rename(_id, **kwargs) - @expose() def top(self, _id): query_params = request.query_string return self.container.top(_id, query_params=query_params) - @expose(generic=True) - def update(self, _id): - abort(404) - - @update.when(method="POST") - def update_POST(self, _id, **kwargs): + def update(self, _id, **kwargs): return self.container.update(_id, kwargs) - @expose(generic=True) def pause(self, _id): - abort(404) - - @pause.when(method="POST") - def pause_POST(self, _id): return self.container.pause(_id) - @expose(generic=True) def unpause(self, _id): - abort(404) - - @unpause.when(method="POST") - def unpause_POST(self, _id): return self.container.unpause(_id) - @expose(generic=True) def wait(self, _id): - abort(404) - - @wait.when(method="POST") - def wait_POST(self, _id): return self.container.wait(_id) - def delete(self, _id): - return self.container.delete(_id) - - -class ContainerController(object): - def __init__(self): - pass - - @expose(generic=True) - def create(self, *args): - abort(404) - - @create.when(method="POST") - def create_container(self, **args): - return container.Container().create(args) - - @expose() - def json(self): - return Container().json() - - @expose(generic=True) - def index(self): - abort(404) - - @expose() - def index_DELETE(self, _id, **kwargs): - Container().delete(_id, **kwargs) - - @expose() - def _lookup(self, id_name_op=None, op=None, *args): - new_url = [] - if op: - new_url.append(op) - elif id_name_op: - new_url.append(id_name_op) - - if op: - new_url.append(id_name_op) - - if args: - new_url.append(args) - - if new_url: - new_url = tuple(new_url) - else: - new_url = tuple(['']) - - return Container(), new_url + def create(self, **args): + return self.container.create(args) diff --git a/dockyard/controllers/v1/image.py b/dockyard/controllers/v1/image.py index f9db886..f9ff25f 100644 --- a/dockyard/controllers/v1/image.py +++ b/dockyard/controllers/v1/image.py @@ -1,70 +1,70 @@ -from pecan import expose, route, abort +from pecan import expose, route +from pecan.rest import RestController from dockyard.common.image import image -class Image(object): +class ImageController(RestController): def __init__(self): self.image = image.Image() - super(Image, self).__init__() @expose() - def push(self, _id): - return self.image.push(_id) + def get_one(self, name_or_id, operation): + if not operation: + operation = name_or_id + name_or_id = None - @expose(generic=True) - def index(self): - abort(404) + return getattr(self, operation)(name_or_id) - @expose(generic=True) - def create(self, *args): - abort(404) + @expose() + # This method is not working, it has to be corrected. + def get(self, operation): + return getattr(self, operation)() - @create.when(method="POST") - def create_POST(self, fromImage, tag='latest'): - return self.image.create(fromImage, tag).data + def search(self, term): + return self.search(term=name_or_id) - @index.when(method="DELETE") - def delete(self, _id): - return self.image.delete(_id).data + def history(self, name_or_id): + return self.history(name_or_id) - @expose() - def json(self, _id=None): - return self.image.list(_id) + def json(self, name_or_id=None): + return self.list_(_id=name_or_id) - @expose() - def search(self, term=None): - return self.image.search(term) + def push(self, _id): + return self.image.push(_id) @expose() - def history(self, _id=None): - return self.image.history(_id) - - @expose(generic=True) - def tag(self, *args): - abort(404) + def post(self, name=None, operation=None): + if not operation: + operation = name_or_id + name_or_id = None - @tag.when(method="POST") - def tag_POST(self, id_or_name, **kwargs): - return self.image.tag(id_or_name, kwargs).data + return getattr(self, operation)(name_or_id) - @expose(generic=True) - def build(self, *args): - abort(404) + # Test this method + def create(self, fromImage, tag='latest'): + return self.image.create(fromImage, tag) - @build.when(method="POST") - def build_POST(self, **kwargs): - return self.image.build(kwargs).data + def tag(self, name, **kwargs): + return self.image.tag(id_or_name, kwargs) + # This method has to be put in the misc section, as per API + # it does not seems to belong to images + def build(self, **kwargs): + return self.image.build(kwargs) -class ImageController(object): - def __init__(self): - pass + def push(self): + return self.push(name) @expose() - def _lookup(self, _id, op=None): - if op is not None: - new_url = (op, _id) - else: - new_url = tuple([_id]) - return Image(), new_url + def delete(self, name): + return self.image.delete(name) + + def list_(self, _id=None): + return self.image.list(_id) + + def search(self, term=None): + return self.image.search(term) + + def history(self, _id=None): + return self.image.history(_id) diff --git a/dockyard/controllers/v1/information.py b/dockyard/controllers/v1/information.py index a3030c3..efc239f 100644 --- a/dockyard/controllers/v1/information.py +++ b/dockyard/controllers/v1/information.py @@ -1,21 +1,22 @@ from pecan import expose +from pecan.rest import RestController from dockyard.common.information import information -class Information(object): +class Information(RestController): def __init__(self): self.information = information.Information() @expose() - def index(self): + def get_all(self): return self.information.info() -class Version(object): +class Version(RestController): def __init__(self): self.information = information.Version() @expose() - def index(self): + def get_all(self): return self.information.version() diff --git a/dockyard/controllers/v1/network.py b/dockyard/controllers/v1/network.py index 3927647..ba700a5 100644 --- a/dockyard/controllers/v1/network.py +++ b/dockyard/controllers/v1/network.py @@ -1,71 +1,41 @@ from pecan import expose, abort +from pecan.rest import RestController from dockyard.common.network import network -class Network(object): +class NetworkController(RestController): def __init__(self): self.network = network.Network() - @expose(generic=True) - def index(self, name_or_id=None): - return self.network.list(name_or_id) + @expose() + def post(self, _id, operation=None, **kwargs): + if not operation: + operation = _id + _id = None - @index.when(method="DELETE") - def index_delete(self, name_or_id): - return self.network.delete(name_or_id) + return getattr(self, operation)(_id=_id, **kwargs) - @expose(generic=True) - def connect(self): - abort(404) + def create(self, _id=None, **kwargs): + return self.network.create(**kwargs) - @connect.when(method='POST') - def connect_(self, _id, **kwargs): + def connect(self, _id=None, **kwargs): return self.network.connect(_id, **kwargs) - @expose(generic=True) - def disconnect(self): - abort(404) - - @disconnect.when(method='POST') - def disconnect_(self, _id, **kwargs): - return self.network.disconnect(_id, **kwargs) - - @expose(generic=True) - def floatingip(self): - abort(404) - - @floatingip.when(method='POST') - def floatingip_(self, _id, **kwargs): - return self.network.attach_floatingip(_id, kwargs) - - -class NetworkController(object): - def __init__(self): - self.network = network.Network() - - @expose(generic=True) - def create(self): - abort(404) + @expose() + def get_one(self, name_or_id): + return self.network.list(name_or_id) - @create.when(method="POST") - def _create(self, **kwargs): - return self.network.create(kwargs) + @expose() + def get(self): + return self.network.list() @expose() - def _lookup(self, id_name_op=None, op=None): + def delete(self, name_or_id): + return self.network.delete(name_or_id) - new_url = [] - if op: - new_url.append(op) - new_url.append(id_name_op) - elif id_name_op: - new_url.append('') - new_url.append(id_name_op) + def disconnect(self, _id, **kwargs): + return self.network.disconnect(_id, **kwargs) - if new_url: - new_url = tuple(new_url) - else: - new_url = tuple(['']) - - return Network(), new_url + def floatingip(self, _id, **kwargs): + return self.network.attach_floatingip(_id, kwargs) diff --git a/dockyard/controllers/v1/volume.py b/dockyard/controllers/v1/volume.py index 3cedf9d..40ee6cd 100644 --- a/dockyard/controllers/v1/volume.py +++ b/dockyard/controllers/v1/volume.py @@ -1,41 +1,27 @@ -from pecan import expose +from pecan import expose, request +from pecan.rest import RestController from dockyard.common.volume import volume - -class Volume(object): +class VolumeController(RestController): + """This class exposes all the API's related with volume. + """ def __init__(self): self.volume = volume.Volume() - @expose(generic=True) - def index(self, name=None): - return self.volume.list(name) - - @index.when(method='DELETE') - def index_DELETE(self, name): - return self.volume.delete(name) - - @expose(generic=True) - def create(self): - abort(404) - - @create.when(method='POST') - def _create(self, **kwargs): - return self.volume.create(kwargs) - + @expose() + def get_one(self, name=None): + return self.volume.list(name=name) -class VolumeController(object): - def __init__(self): - pass + @expose() + def get(self): + return self.volume.list() @expose() - def _lookup(self, op=None): - new_url = [''] - if op: - new_url.append(op) + def delete(self, name): + return self.volume.delete(name=name) - if new_url: - new_url = tuple(new_url) - else: - new_url = tuple(['']) - return Volume(), new_url + @expose() + def post(self, operation=None): + body = request.body + return self.volume.create(data=body) diff --git a/test/floatingip.py b/test/floatingip.py index a204064..58a688b 100644 --- a/test/floatingip.py +++ b/test/floatingip.py @@ -1 +1 @@ -curl http://127.0.0.1:5869/dockyard/networks/floatingip/7289 -XPOST -d '{"psid": "7289", "ip": "10.0.9.2", "mask": "24", "gateway": "10.0.9.1"}' +curl http://127.0.0.1:5869/dockyard/networks/floatingip/4162 -XPOST -d '{"psid": "4162", "ip": "10.0.9.2", "mask": "24", "gateway": "10.0.9.1"}' From b25a26c7ff907c80bda6f7be5f761bd534b77986 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sat, 11 Jun 2016 01:24:26 +0530 Subject: [PATCH 49/62] Added check for consul servics --- dockyard/common/membership/consul/consul_driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dockyard/common/membership/consul/consul_driver.py b/dockyard/common/membership/consul/consul_driver.py index f8fb1f6..7610bc3 100644 --- a/dockyard/common/membership/consul/consul_driver.py +++ b/dockyard/common/membership/consul/consul_driver.py @@ -29,10 +29,13 @@ def _register_service(self, name, host, port, tags=None): raise exception.Incompleteinfo(message) + http = ("http://%s:%d" % (host, port)) self.consul.agent.service.register(name=name, address=host, port = port, - tags = tags) + tags = tags, + http = http, + interval=10) def _register_dockyard(self): ip = CONF['default']['host'] From 3e31fb68ba8f2fb20e28d4c561a6e7a70e6c5f71 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Tue, 14 Jun 2016 18:11:10 +0530 Subject: [PATCH 50/62] Changes in consul driver --- dockyard/common/membership/consul/consul_driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dockyard/common/membership/consul/consul_driver.py b/dockyard/common/membership/consul/consul_driver.py index 7610bc3..e27dcbb 100644 --- a/dockyard/common/membership/consul/consul_driver.py +++ b/dockyard/common/membership/consul/consul_driver.py @@ -28,8 +28,11 @@ def _register_service(self, name, host, port, tags=None): LOG.exception("Cannot continue, Incomplete info: %s" % message) raise exception.Incompleteinfo(message) + if host: + http = ("http://%s:%d" % (host, port)) + else: + http = ("http://localhost:%d" % (port)) - http = ("http://%s:%d" % (host, port)) self.consul.agent.service.register(name=name, address=host, port = port, From e14fcfb0cc7d3643c31f02ea612c11131e637cce Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Tue, 14 Jun 2016 20:22:36 +0530 Subject: [PATCH 51/62] Changes for consul to return only up nodes --- .../common/membership/consul/consul_driver.py | 41 +++++++++++++------ requirement.txt | 2 + 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/dockyard/common/membership/consul/consul_driver.py b/dockyard/common/membership/consul/consul_driver.py index e27dcbb..af0cfb4 100644 --- a/dockyard/common/membership/consul/consul_driver.py +++ b/dockyard/common/membership/consul/consul_driver.py @@ -1,4 +1,5 @@ import consul +import netifaces from oslo_config import cfg from oslo_log import log as logging @@ -9,30 +10,44 @@ LOG = logging.getLogger(__name__) + +def get_localhost_ip(): + """This method resturns localhost ip. + """ + ifs = netifaces.interfaces() + for i in ifs: + try: + addr = netifaces.ifaddresses(i)[netifaces.AF_INET][0]['addr'] + except KeyError: + pass + + if addr == '127.0.0.1': + continue + + yield addr + + class Consul(Membership): def __init__(self): self.consul = consul.Consul() - def _register_service(self, name, host, port, tags=None): + def _register_service(self, name, host, port, tags=None, url=''): if not name: message = ('Service name to use for registering') LOG.exception("Cannot continue, Incomplete info: %s" % message) raise exception.IncompleteInfo(message) -# if not host: -# message = ('IP address for the service to be used') -# raise exception.IncompleteInfo(message) - if not port: message = ('Port number used by the services to listen') LOG.exception("Cannot continue, Incomplete info: %s" % message) raise exception.Incompleteinfo(message) - if host: - http = ("http://%s:%d" % (host, port)) - else: - http = ("http://localhost:%d" % (port)) + if not host: + for ip in get_localhost_ip(): + host = ip + break + http = ("http://%s:%d/%s" % (host, port, url)) self.consul.agent.service.register(name=name, address=host, port = port, @@ -45,11 +60,11 @@ def _register_dockyard(self): if ip == '0.0.0.0': ip = None - + port = CONF['default']['port'] tags = ['master'] name = CONF['consul']['service_name'] - + self._register_service(name, ip, port, tags) def _register_docker(self): @@ -59,8 +74,8 @@ def _register_docker(self): port = CONF['docker']['docker_port'] name = CONF['docker']['docker_name'] - - self._register_service(name, ip, port) + + self._register_service(name, ip, port, url='images/json') def register(self): diff --git a/requirement.txt b/requirement.txt index 043919b..eb88aa7 100644 --- a/requirement.txt +++ b/requirement.txt @@ -4,3 +4,5 @@ pecan python-consul nsenter pyroute2 +netifaces +python-dev From 36fc8a31180374440050d9e809a733c0e2aea5d0 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sat, 18 Jun 2016 15:08:29 +0530 Subject: [PATCH 52/62] Fixed issue #10 --- .../common/membership/consul/consul_driver.py | 20 ++++++++++++++++--- dockyard/common/membership/consul/utils.py | 14 +++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 dockyard/common/membership/consul/utils.py diff --git a/dockyard/common/membership/consul/consul_driver.py b/dockyard/common/membership/consul/consul_driver.py index af0cfb4..1f45276 100644 --- a/dockyard/common/membership/consul/consul_driver.py +++ b/dockyard/common/membership/consul/consul_driver.py @@ -5,6 +5,7 @@ from dockyard.common.membership.base import Membership from dockyard.common import exception +from dockyard.common.membership.consul import utils as consul_utils CONF = cfg.CONF @@ -26,10 +27,21 @@ def get_localhost_ip(): yield addr +class ConsulHealthCheck(object): + def __init__(self): + self.healthy = consul.Consul().health + + def get_healthy_nodes(self, service="dockyard"): + """Get healthy nodes in the cluster. + """ + services = self.healthy.service(service=service, passing=True) + return consul_utils.get_formatted_hosts(services) + class Consul(Membership): def __init__(self): self.consul = consul.Consul() + self.healthy_services = ConsulHealthCheck() def _register_service(self, name, host, port, tags=None, url=''): if not name: @@ -53,7 +65,7 @@ def _register_service(self, name, host, port, tags=None, url=''): port = port, tags = tags, http = http, - interval=10) + interval=15) def _register_dockyard(self): ip = CONF['default']['host'] @@ -117,9 +129,11 @@ def _get_services(self, services): def get_all_hosts(self, tag='agent'): """Returns all the members current agent sees. """ - services = self.consul.catalog.service('docker') + # services = self.consul.catalog.service('docker') + services = self.healthy_services.get_healthy_nodes() if not services: message = "No services are registered to the consul" raise exception.NoValidHostFound(message) - return self._get_services(services) + print services + return services diff --git a/dockyard/common/membership/consul/utils.py b/dockyard/common/membership/consul/utils.py new file mode 100644 index 0000000..6ee7974 --- /dev/null +++ b/dockyard/common/membership/consul/utils.py @@ -0,0 +1,14 @@ +# This module contains utility required by consul driver only. + +def get_formatted_hosts(services): + """This method is responsible for formatting output as per + need of the dockyard. + """ + healthy_services = [] + for service in services[1]: + service_info = dict() + service_info["host"] = str(service["Service"]["Address"]) + service_info["port"] = service["Service"]["Port"] + healthy_services.append(service_info) + + return healthy_services From e72898f93351cb23b164da79b246154405da1c83 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sun, 19 Jun 2016 18:37:44 +0530 Subject: [PATCH 53/62] Fixes for Providing storage capablity --- dockyard/common/container/container.py | 89 ++++++++++--------- dockyard/common/utils.py | 10 ++- dockyard/engine/common/containers/__init__.py | 0 .../common/containers/store/__init__.py | 0 .../containers/store/consul/__init__.py | 40 +++++++++ .../containers/store/consul/consul_client.py | 11 +++ .../containers/store/consul/synchronizer.py | 47 ++++++++++ dockyard/engine/controllers/__init__.py | 2 + etc/dockyard/dockyard.conf | 5 +- 9 files changed, 159 insertions(+), 45 deletions(-) create mode 100644 dockyard/engine/common/containers/__init__.py create mode 100644 dockyard/engine/common/containers/store/__init__.py create mode 100644 dockyard/engine/common/containers/store/consul/__init__.py create mode 100644 dockyard/engine/common/containers/store/consul/consul_client.py create mode 100644 dockyard/engine/common/containers/store/consul/synchronizer.py diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index b17b1c5..55445de 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -14,56 +14,61 @@ class Container(object): def __init__(self): self.url = url.URL(self.base_url) - def list(self, name_or_id=None, query_params=None): + def list(self, name_or_id=None, query_params=None, host=None): url_ = self.url.make_url(url_='json', id_=name_or_id) msg = ("List containers with url: %s " "query_params: %s" % (url_, query_params)) LOG.debug(msg) - return utils.dispatch_get_request(url_, query_params=query_params) + return utils.dispatch_get_request(url_, query_params=query_params, + host=host) - def changes(self, name_or_id=None): + def changes(self, name_or_id=None, host=None): url_ = self.url.make_url(url_='changes', id_=name_or_id) - return utils.dispatch_get_request(url_) + return utils.dispatch_get_request(url_, host=host) - def resize(self, id_, **kwargs): + def resize(self, id_, host=None, **kwargs): url_ = self.url.make_url(url_='resize', id_=id_) query = '' if kwargs: query = link.make_query_url(kwargs) - return utils.dispatch_get_request(url_, query_params=query) + return utils.dispatch_get_request(url_, query_params=query, host=host) - def export(self, name_or_id=None): + def export(self, name_or_id=None, host=None): url_ = self.url.make_url(url_='export', id_=name_or_id) redirect(utils.get_link(url_)) - def top(self, name_or_id=None, query_params=None): + def top(self, name_or_id=None, query_params=None, host=None): url_ = self.url.make_url(url_='top', id_=name_or_id) - return utils.dispatch_get_request(url_, query_params=query_params) + return utils.dispatch_get_request(url_, query_params=query_params, + host=host) - def stats(self, id_): + def stats(self, id_, host=None): url_ = self.url.make_url(url_='stats', id_=id_) redirect(utils.get_link(url_)) - def archive(self, id_, query_params=None): + def archive(self, id_, query_params=None, host=None): url_ = self.url.make_url(url_='archive', id_=id_) - return utils.dispatch_put_request(url_, query_params=query_params) + return utils.dispatch_put_request(url_, query_params=query_params, + host=host) - def create(self, body=None): + def create(self, body=None, host=None): url_ = self.url.make_url(url_='create') - return utils.dispatch_post_request(url_, body=json.dumps(body)) + return utils.dispatch_post_request(url_, body=json.dumps(body), + host=host) - def upload(self, id_, body=None, query=None, **kwargs): + def upload(self, id_, body=None, query=None, host=None, **kwargs): url_ = self.url.make_url(url_='archive', id_=id_) - return utils.dispatch_put_request(url_, body=body, query_params=query) + return utils.dispatch_put_request(url_, body=body, query_params=query, + host=host) - def copy(self, id_): + def copy(self, id_, host=None): url_ = self.url.make_url(url_='copy', id_=id_) - redirect(utils.get_link(url_)) + redirect(utils.get_link(url_, host=host)) - def logs(self, id_, query_string, **kwargs): + def logs(self, id_, query_string, host=None, **kwargs): url_ = self.url.make_url(url_='logs', id_=id_) query = '' @@ -72,59 +77,61 @@ def logs(self, id_, query_string, **kwargs): redirect(utils.get_link(url_, query_params=query)) - def start(self, id_, query_params=None): + def start(self, id_, query_params=None, host=None): url_ = self.url.make_url(url_='start', id_=id_) - return utils.dispatch_post_request(url_, query_params=query_params) + return utils.dispatch_post_request(url_, query_params=query_params, + host=host) - def restart(self, id_, query_params=None): + def restart(self, id_, query_params=None, host=None): url_ = self.url.make_url(url_='restart', id_=id_) - return utils.dispatch_post_request(url_, query_params=query_params) + return utils.dispatch_post_request(url_, query_params=query_params, + host=host) - def kill(self, id_, query_params=None): + def kill(self, id_, query_params=None, host=None): url_ = self.url.make_url(url_='kill', id_=id_) - return utils.dispatch_post_request(url_, query_params=query_params) + return utils.dispatch_post_request(url_, query_params=query_params, host=host) - def stop(self, id_, query_params=None): + def stop(self, id_, query_params=None, host=None): url_ = self.url.make_url(url_='stop', id_=id_) - return utils.dispatch_post_request(url_, query_params=query_params) + return utils.dispatch_post_request(url_, query_params=query_params, host=host) def exe(self, _id): abort(404) - def attach(self, id_, query_params=None): + def attach(self, id_, query_params=None, host=None): url_ = self.url.make_url(url_='attach', id_=id_) - redirect(utils.get_link(url_, quer_params=query_params)) + redirect(utils.get_link(url_, quer_params=query_params, host=host)) - def rename(self, id_, **kwargs): + def rename(self, id_, host=None, **kwargs): url_ = self.url.make_url(url_='rename', id_=id_) query = '' if kwargs: query = link.make_query_url(kwargs) - return utils.dispatch_post_request(url_, query_params=query) + return utils.dispatch_post_request(url_, query_params=query, host=host) - def update(self, id_, body=None): + def update(self, id_, body=None, host=None): url_ = self.url.make_url(url_='update', id_=id_) - return utils.dispatch_post_request(url_, body=json.dumps(body)) + return utils.dispatch_post_request(url_, body=json.dumps(body), host=host) - def pause(self, id_): + def pause(self, id_, host=None): url_ = self.url.make_url(url_='pause', id_=id_) - return utils.dispatch_post_request(url_) + return utils.dispatch_post_request(url_, host=host) - def unpause(self, id_): + def unpause(self, id_, host=None): url_ = self.url.make_url(url_='unpause', id_=id_) - return utils.dispatch_post_request(url_) + return utils.dispatch_post_request(url_, host=host) - def wait(self, id_): + def wait(self, id_, host=None): url_ = self.url.make_url(url_='wait', id_=id_) - return utils.dispatch_post_request(url_) + return utils.dispatch_post_request(url_, host=host) - def delete(self, id_, **kwargs): + def delete(self, id_, host=None, **kwargs): url_ = self.url.make_url(id_=id_) query = '' if kwargs: query = link.make_query_url(kwargs) - return utils.dispatch_delete_request(url_, query_params=query) + return utils.dispatch_delete_request(url_, query_params=query, host=host) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index ff93e58..305421e 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -105,8 +105,10 @@ def prepare_logging(argv=None): CONF(argv[1:], project='dockyard') logging.setup(CONF, 'dockyard') -def dispatch_get_request(url, headers=None, protocol='http', query_params=None): - ln = get_link(url=url, protocol=protocol) +def dispatch_get_request(url, headers=None, protocol='http', + query_params=None, host=None): + + ln = get_link(host=host, url=url, protocol=protocol) if query_params: query = link.make_query_url(query_params) @@ -127,7 +129,9 @@ def dispatch_post_request(url, host=None, protocol='http', return dispatch_post_req(url=ln, post_params=query_params, body=body) -def dispatch_put_request(url, protocol='http', body=None, query_params=None): +def dispatch_put_request(url, protocol='http', body=None, + host=None, query_params=None): + ln = get_link(url=url, protocol=protocol) if query_params: diff --git a/dockyard/engine/common/containers/__init__.py b/dockyard/engine/common/containers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine/common/containers/store/__init__.py b/dockyard/engine/common/containers/store/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine/common/containers/store/consul/__init__.py b/dockyard/engine/common/containers/store/consul/__init__.py new file mode 100644 index 0000000..fbb5f9c --- /dev/null +++ b/dockyard/engine/common/containers/store/consul/__init__.py @@ -0,0 +1,40 @@ +from oslo_config import cfg +from threading import Thread +import time + +from dockyard.engine.common.containers.store.consul.synchronizer import ( + ContainerSynchronizer) + +CONSUL_DATABASE_OPT = [ + cfg.IntOpt('synchronization_time', + default=20, + help='Time after which data will be synced .'), +] + +CONF = cfg.CONF +opt_group = cfg.OptGroup(name='consul_database', + title='Group for the configuration options ' + 'for consul as database.') +CONF.register_group(opt_group) +CONF.register_opts(CONSUL_DATABASE_OPT, opt_group) + + + +class Synchronizer(Thread): + def __init__(self): + Thread.__init__(self) + self.container = ContainerSynchronizer() + self.sleep_time = CONF.consul_database.synchronization_time + + + def run(self): + """This thread is responsible for synchronizations of containers, + and other docker resources. + """ + self.container.synchronize() + time.sleep(self.sleep_time) + + +sync = Synchronizer() +sync.setName("Synchronizer") +sync.start() diff --git a/dockyard/engine/common/containers/store/consul/consul_client.py b/dockyard/engine/common/containers/store/consul/consul_client.py new file mode 100644 index 0000000..81aee73 --- /dev/null +++ b/dockyard/engine/common/containers/store/consul/consul_client.py @@ -0,0 +1,11 @@ +import consul + +class ConsulKV(object): + def __init__(self): + self.consul = consul.Consul() + + def get(self, key): + self.consul.kv.get(key) + + def get(self, key, value): + self.consul.kv.put(key, value) diff --git a/dockyard/engine/common/containers/store/consul/synchronizer.py b/dockyard/engine/common/containers/store/consul/synchronizer.py new file mode 100644 index 0000000..18ec42e --- /dev/null +++ b/dockyard/engine/common/containers/store/consul/synchronizer.py @@ -0,0 +1,47 @@ +import ast +from oslo_config import cfg + +from dockyard.common.container.container import Container +from dockyard.engine.common.containers.store.consul import consul_client + + +CONF = cfg.CONF + + +class ContainerSynchronizer(object): + def __init__(self): + self.host = { + 'host': CONF.docker.docker_host, + 'port': CONF.docker.docker_port + } + + self.container = Container() + self.db = consul_client.ConsulKV() + + def _containers(self): + """This method fetch all the containers running on local machines. + """ + containers = self.container.list() + containers = ast.literal_eval(containers) + for con in containers: + yield con + + + def synchronize(self): + """This method is responsible for initilizing databases at the start + of dockyard. It collects all the containers running on localhost + and putt the data in the consul + """ + for container in self._containers(): + self._register(container) + + def _inspect(self, id_): + return self.container.list(name_or_id=id_, host=self.host) + + def _register(self, container): + """This method is responsible for registering a container. + """ + id_ = container["Id"] + container_info = self.container._inspect(id_=id_) + print container_info + self.db.put(id_, container_info) diff --git a/dockyard/engine/controllers/__init__.py b/dockyard/engine/controllers/__init__.py index 692f9ea..1321a4b 100644 --- a/dockyard/engine/controllers/__init__.py +++ b/dockyard/engine/controllers/__init__.py @@ -1,5 +1,7 @@ from oslo_config import cfg +import dockyard.engine.common.containers.store.consul + NETWORK_DRIVER_OPT = [ cfg.StrOpt('network_driver', default='bridges.linux.LinuxBridgeManager', diff --git a/etc/dockyard/dockyard.conf b/etc/dockyard/dockyard.conf index 160df59..c1da802 100644 --- a/etc/dockyard/dockyard.conf +++ b/etc/dockyard/dockyard.conf @@ -41,4 +41,7 @@ docker_port = 2375 # service name docker_name = 'docker' - +[consul_database] +# This is the time after which synchronizer thread will wake up and start +# syncing containers launched. +synchronization_time = 20 From bdb4806f9909e4a42fa903c02aa17da4b9882803 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Wed, 22 Jun 2016 17:06:45 +0530 Subject: [PATCH 54/62] Changes for consul as database --- dockyard/common/base.py | 20 ++----- dockyard/common/container/container.py | 3 ++ dockyard/common/utils.py | 52 +++++++++++++------ .../containers/store/consul/__init__.py | 1 - .../containers/store/consul/synchronizer.py | 18 ++++--- dockyard/engine/common/utils.py | 16 ++++++ 6 files changed, 69 insertions(+), 41 deletions(-) diff --git a/dockyard/common/base.py b/dockyard/common/base.py index 1e6e01d..b67214e 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -19,12 +19,11 @@ engine_client_loc = 'dockyard.engine_client' engine_client_info = (('%s.%s') % (engine_client_loc, engine_client_info)) module_name, class_name = engine_client_info.rsplit(".", 1) -class_ = getattr(importlib.import_module(module_name), class_name) -engine_client = class_() +engine_client = getattr(importlib.import_module(module_name), class_name) class URL(object): def __init__(self): - self.engine_client = engine_client() + pass @abc.abstractmethod def send(self, method, url, headers=None, post_params=None, @@ -39,6 +38,7 @@ def send(self, method, url, headers=None, post_params=None, class DockyardURL(URL): def __init__(self): + self.engine_client = engine_client() self.pool = PoolManager() def _is_local_request(self): @@ -48,7 +48,7 @@ def _is_local_request(self): request.headers.environ['Request-Status'] except KeyError: status = False - else: + except: status = True return status @@ -77,15 +77,3 @@ def send(self, method, url, headers=None, post_params=None, #self.engine_client.post_process(method=method, url=url, # body=body, headers=headers) return data - - - def _add_headers(self, headers): - """This method adds header to each request. - Valid values for Request-Status header are Scheduled. - More values will be added, as per requirements. - """ - if not headers: - headers = dict() - - headers['Request-Status'] = 'Scheduled' - return headers diff --git a/dockyard/common/container/container.py b/dockyard/common/container/container.py index 55445de..0f09128 100644 --- a/dockyard/common/container/container.py +++ b/dockyard/common/container/container.py @@ -16,10 +16,13 @@ def __init__(self): def list(self, name_or_id=None, query_params=None, host=None): url_ = self.url.make_url(url_='json', id_=name_or_id) + msg = ("List containers with url: %s " "query_params: %s" % (url_, query_params)) + LOG.debug(msg) + return utils.dispatch_get_request(url_, query_params=query_params, host=host) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index 305421e..821dd3c 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -88,10 +88,21 @@ def get_localhost(): def get_link(url, host=None, protocol='http'): + headers = dict() + if not host: host = get_host() + headers['Request-Status'] = 'Scheduled' + + url = link.make_url(host=host['host'], port=host['port'], url=url) + return (url, headers) - return link.make_url(host=host['host'], port=host['port'], url=url) +def merge_dict(x=dict(), y=dict()): + """Merge two dictionaries. + """ + z = x.copy() + z.update(y) + return z def prepare_logging(argv=None): """ @@ -105,58 +116,65 @@ def prepare_logging(argv=None): CONF(argv[1:], project='dockyard') logging.setup(CONF, 'dockyard') -def dispatch_get_request(url, headers=None, protocol='http', +def dispatch_get_request(url, headers=dict(), protocol='http', query_params=None, host=None): - ln = get_link(host=host, url=url, protocol=protocol) + (ln, hdr) = get_link(host=host, url=url, protocol=protocol) if query_params: query = link.make_query_url(query_params) ln = (('%s?%s') % (ln, query)) - return request.send(method='GET', url=ln) + headers = merge_dict(headers, hdr) + return request.send(method='GET', url=ln, headers=headers) def dispatch_post_request(url, host=None, protocol='http', body=None, query_params=None): - ln = get_link(host=host, url=url, protocol=protocol) + (ln, hdr) = get_link(host=host, url=url, protocol=protocol) if query_params: query = link.make_query_url(query_params) ln = (('%s?%s') % (ln, query)) - return dispatch_post_req(url=ln, post_params=query_params, body=body) + headers = merge_dict(headers, hdr) + return dispatch_post_req(url=ln, post_params=query_params, + body=body, headers=headers) def dispatch_put_request(url, protocol='http', body=None, host=None, query_params=None): - ln = get_link(url=url, protocol=protocol) + (ln, hdr) = get_link(host=host, url=url, protocol=protocol) if query_params: query = link.make_query_url(query_params) ln = (('%s?%s') % (ln, query)) - return dispatch_put_req(url=ln, post_params=query_params, body=body) + headers = merge_dict(headers, hdr) + return dispatch_put_req(url=ln, post_params=query_params, + body=body, headers=headers) -def dispatch_delete_request(url, headers = None, protocol='http', - query_params=None): - ln = get_link(url=url, protocol=protocol) - return request.send(method='DELETE', url=ln) +def dispatch_delete_request(url, headers=dict(), protocol='http', + query_params=None, host=None): + (ln, hdr) = get_link(url=url, protocol=protocol, host=None) + headers = merge_dict(headers, hdr) + return request.send(method='DELETE', url=ln, headers=headers) -def dispatch_post_req(url, headers=None, body=None, - post_params=None, host=None): +def dispatch_post_req(url, headers=dict(), body=None, post_params=None): if not headers: - headers = {'Content-Type': 'application/json'} - + hdr = {'Content-Type': 'application/json'} + headers = merge_dict(headers, hdr) + return request.send(method='POST', url=url, headers=headers, body=body) -def dispatch_put_req(url, headers=None, body=None, post_params=None): +def dispatch_put_req(url, headers=dict(), body=None, post_params=None): if not headers: headers = {'Content-Type': 'application/x-tar'} + headers = merge_dict(headers, hdr) return request.send(method='PUT', url=url, headers=headers, body=body) diff --git a/dockyard/engine/common/containers/store/consul/__init__.py b/dockyard/engine/common/containers/store/consul/__init__.py index fbb5f9c..1eed9a4 100644 --- a/dockyard/engine/common/containers/store/consul/__init__.py +++ b/dockyard/engine/common/containers/store/consul/__init__.py @@ -19,7 +19,6 @@ CONF.register_opts(CONSUL_DATABASE_OPT, opt_group) - class Synchronizer(Thread): def __init__(self): Thread.__init__(self) diff --git a/dockyard/engine/common/containers/store/consul/synchronizer.py b/dockyard/engine/common/containers/store/consul/synchronizer.py index 18ec42e..9005acb 100644 --- a/dockyard/engine/common/containers/store/consul/synchronizer.py +++ b/dockyard/engine/common/containers/store/consul/synchronizer.py @@ -1,6 +1,7 @@ import ast from oslo_config import cfg +from dockyard.engine.common.utils import get_localhost_ip from dockyard.common.container.container import Container from dockyard.engine.common.containers.store.consul import consul_client @@ -10,18 +11,22 @@ class ContainerSynchronizer(object): def __init__(self): - self.host = { - 'host': CONF.docker.docker_host, - 'port': CONF.docker.docker_port - } - + if CONF.docker.docker_host == "0.0.0.0": + for ip in get_localhost_ip(): + host = ip + break + else: + host = CONF.docker.docker_host + + port = CONF.docker.docker_port + self.host = { 'host': host, 'port': port } self.container = Container() self.db = consul_client.ConsulKV() def _containers(self): """This method fetch all the containers running on local machines. """ - containers = self.container.list() + containers = self.container.list(host=self.host) containers = ast.literal_eval(containers) for con in containers: yield con @@ -43,5 +48,4 @@ def _register(self, container): """ id_ = container["Id"] container_info = self.container._inspect(id_=id_) - print container_info self.db.put(id_, container_info) diff --git a/dockyard/engine/common/utils.py b/dockyard/engine/common/utils.py index ff9f9f3..9c474c3 100644 --- a/dockyard/engine/common/utils.py +++ b/dockyard/engine/common/utils.py @@ -1,5 +1,6 @@ import ast import json +import netifaces def json_dump(data): """This function dumps data into json format to send over the internet. @@ -10,3 +11,18 @@ def str_to_dict(data): """This function convert string to dict. """ return ast.literal_eval(data) + +def get_localhost_ip(): + """This method resturns localhost ip. + """ + ifs = netifaces.interfaces() + for i in ifs: + try: + addr = netifaces.ifaddresses(i)[netifaces.AF_INET][0]['addr'] + except KeyError: + pass + + if addr == '127.0.0.1': + continue + + yield addr From c56e01d4c9cef5b84be50a3460ad7def8fe4e5e4 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 28 Jun 2016 19:13:32 +0530 Subject: [PATCH 55/62] Problem removed for wait time --- dockyard/cmd/api.py | 31 -------------- dockyard/common/base.py | 15 +++---- .../common/membership/consul/consul_driver.py | 1 - dockyard/common/utils.py | 40 ++++++++++++++++++- .../containers/store/consul/__init__.py | 5 ++- .../containers/store/consul/consul_client.py | 2 +- .../containers/store/consul/synchronizer.py | 7 ++-- dockyard/engine/common/utils.py | 15 ------- 8 files changed, 54 insertions(+), 62 deletions(-) delete mode 100644 dockyard/cmd/api.py diff --git a/dockyard/cmd/api.py b/dockyard/cmd/api.py deleted file mode 100644 index 8f11839..0000000 --- a/dockyard/cmd/api.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import sys - -from oslo_config import cfg -from oslo_log import log as logging -from wsgiref import simple_server - -from dockyard.api import app as api_app -from dockyard.common import utils - - -LOG = logging.getLogger(__name__) - -def main(): - utils.prepare_logging(sys.argv) - - app = api_app.setup_app() - - # create the wsgi server and start it - host, port = cfg.CONF.default.host, cfg.CONF.default.port - srv = simple_server.make_server(host, port, app) - - LOG.info('Starting dockyard in PID %s', os.getpid()) - LOG.debug("Configuration:") - cfg.CONF.log_opt_values(LOG, logging.DEBUG) - - LOG.info('serving at http://%s:%s' % (host, port)) - srv.serve_forever() - - -main() diff --git a/dockyard/common/base.py b/dockyard/common/base.py index b67214e..6004758 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -62,16 +62,17 @@ def send(self, method, url, headers=None, post_params=None, :post_params: post parameters. :body: Request body. """ - req_type = self._is_local_request() + #req_type = self._is_local_request() # Pre processing needs to be done for dockyard feature before # performing some actions # print method, url, headers, body - if req_type: - self.engine_client.pre_process(method=method, url=url, - body=body, headers=headers) - else: - headers = self._add_headers(headers) - + #if req_type: + # self.engine_client.pre_process(method=method, url=url, + # body=body, headers=headers) + #else: + # headers = self._add_headers(headers) + headers = {} + print method, url, headers, body data = self.pool.urlopen(method, url, headers=headers, body=body).data # Post processing needs to be done for some of the dockyard operations #self.engine_client.post_process(method=method, url=url, diff --git a/dockyard/common/membership/consul/consul_driver.py b/dockyard/common/membership/consul/consul_driver.py index 1f45276..1279b4c 100644 --- a/dockyard/common/membership/consul/consul_driver.py +++ b/dockyard/common/membership/consul/consul_driver.py @@ -135,5 +135,4 @@ def get_all_hosts(self, tag='agent'): message = "No services are registered to the consul" raise exception.NoValidHostFound(message) - print services return services diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index 821dd3c..e214dec 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -1,4 +1,5 @@ import importlib +import netifaces from oslo_config import cfg from oslo_log import log as logging from pecan import request as rcvd_req @@ -65,16 +66,38 @@ def get_host(): If this instance of the process is engine then it should direct to docker procss running on local machine and does preprocessing. """ + host = '' try: rcvd_req.headers.environ['Request-Status'] except KeyError: hosts = membership.get_all_hosts() host = scheduler.get_host(hosts=hosts) - else: - host = { 'host': CONF['docker_host'], 'port': CONF['docker_port'] } + + if (not host) or is_localhost(host=host): + host = { + 'host': CONF['docker']['docker_host'], + 'port': CONF['docker']['docker_port'] + } return host + +def get_localhost_ip(): + """This method resturns localhost ip. + """ + ifs = netifaces.interfaces() + for i in ifs: + try: + addr = netifaces.ifaddresses(i)[netifaces.AF_INET][0]['addr'] + except KeyError: + pass + + if addr == '127.0.0.1': + continue + + yield addr + + def get_localhost(): d = dict() @@ -87,6 +110,19 @@ def get_localhost(): return d +def is_localhost(host): + """This method returns whether specified host is local machine's IP + or not. + """ + localhost_ips = get_localhost_ip() + for ip in localhost_ips: + if ip == host['host']: + return True + print ip + + return False + + def get_link(url, host=None, protocol='http'): headers = dict() diff --git a/dockyard/engine/common/containers/store/consul/__init__.py b/dockyard/engine/common/containers/store/consul/__init__.py index 1eed9a4..6de8f25 100644 --- a/dockyard/engine/common/containers/store/consul/__init__.py +++ b/dockyard/engine/common/containers/store/consul/__init__.py @@ -30,8 +30,9 @@ def run(self): """This thread is responsible for synchronizations of containers, and other docker resources. """ - self.container.synchronize() - time.sleep(self.sleep_time) + while True: + self.container.synchronize() + time.sleep(self.sleep_time) sync = Synchronizer() diff --git a/dockyard/engine/common/containers/store/consul/consul_client.py b/dockyard/engine/common/containers/store/consul/consul_client.py index 81aee73..a97d01e 100644 --- a/dockyard/engine/common/containers/store/consul/consul_client.py +++ b/dockyard/engine/common/containers/store/consul/consul_client.py @@ -7,5 +7,5 @@ def __init__(self): def get(self, key): self.consul.kv.get(key) - def get(self, key, value): + def put(self, key, value): self.consul.kv.put(key, value) diff --git a/dockyard/engine/common/containers/store/consul/synchronizer.py b/dockyard/engine/common/containers/store/consul/synchronizer.py index 9005acb..44ecff2 100644 --- a/dockyard/engine/common/containers/store/consul/synchronizer.py +++ b/dockyard/engine/common/containers/store/consul/synchronizer.py @@ -1,7 +1,7 @@ import ast from oslo_config import cfg -from dockyard.engine.common.utils import get_localhost_ip +from dockyard.common.utils import get_localhost_ip from dockyard.common.container.container import Container from dockyard.engine.common.containers.store.consul import consul_client @@ -47,5 +47,6 @@ def _register(self, container): """This method is responsible for registering a container. """ id_ = container["Id"] - container_info = self.container._inspect(id_=id_) - self.db.put(id_, container_info) + container_info = self._inspect(id_=id_) + key = str((id_, self.host)) + self.db.put(key, container_info) diff --git a/dockyard/engine/common/utils.py b/dockyard/engine/common/utils.py index 9c474c3..69383be 100644 --- a/dockyard/engine/common/utils.py +++ b/dockyard/engine/common/utils.py @@ -11,18 +11,3 @@ def str_to_dict(data): """This function convert string to dict. """ return ast.literal_eval(data) - -def get_localhost_ip(): - """This method resturns localhost ip. - """ - ifs = netifaces.interfaces() - for i in ifs: - try: - addr = netifaces.ifaddresses(i)[netifaces.AF_INET][0]['addr'] - except KeyError: - pass - - if addr == '127.0.0.1': - continue - - yield addr From 407fc4b4626fb68fb9c93b302417d7c10b3cc0a5 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Tue, 28 Jun 2016 19:26:15 +0530 Subject: [PATCH 56/62] Added code again --- dockyard/cmd/api.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 dockyard/cmd/api.py diff --git a/dockyard/cmd/api.py b/dockyard/cmd/api.py new file mode 100644 index 0000000..8f11839 --- /dev/null +++ b/dockyard/cmd/api.py @@ -0,0 +1,31 @@ +import os +import sys + +from oslo_config import cfg +from oslo_log import log as logging +from wsgiref import simple_server + +from dockyard.api import app as api_app +from dockyard.common import utils + + +LOG = logging.getLogger(__name__) + +def main(): + utils.prepare_logging(sys.argv) + + app = api_app.setup_app() + + # create the wsgi server and start it + host, port = cfg.CONF.default.host, cfg.CONF.default.port + srv = simple_server.make_server(host, port, app) + + LOG.info('Starting dockyard in PID %s', os.getpid()) + LOG.debug("Configuration:") + cfg.CONF.log_opt_values(LOG, logging.DEBUG) + + LOG.info('serving at http://%s:%s' % (host, port)) + srv.serve_forever() + + +main() From 99af59d3f111379fd8a4f474348ef53aa1cef645 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Wed, 29 Jun 2016 19:33:03 +0530 Subject: [PATCH 57/62] Scheduling on more than on node --- dockyard/common/base.py | 2 -- dockyard/common/utils.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dockyard/common/base.py b/dockyard/common/base.py index 6004758..9660e2a 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -71,8 +71,6 @@ def send(self, method, url, headers=None, post_params=None, # body=body, headers=headers) #else: # headers = self._add_headers(headers) - headers = {} - print method, url, headers, body data = self.pool.urlopen(method, url, headers=headers, body=body).data # Post processing needs to be done for some of the dockyard operations #self.engine_client.post_process(method=method, url=url, diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index e214dec..ca286d0 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -68,7 +68,7 @@ def get_host(): """ host = '' try: - rcvd_req.headers.environ['Request-Status'] + rcvd_req.headers['Request-Status'] except KeyError: hosts = membership.get_all_hosts() host = scheduler.get_host(hosts=hosts) From 883fe7f7275497560bd1f532ebc9635300df4391 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Sun, 3 Jul 2016 00:01:24 +0530 Subject: [PATCH 58/62] Adding logic for preprocessing and post processing --- dockyard/common/base.py | 17 ++--- .../store => common/stores}/__init__.py | 0 .../stores/consul}/__init__.py | 0 .../stores}/consul/consul_client.py | 4 +- dockyard/common/utils.py | 1 - dockyard/controllers/v1/container.py | 2 +- .../containers/store/consul/__init__.py | 40 ---------- dockyard/engine/controllers/__init__.py | 2 - dockyard/engine_client/__init__.py | 24 ++++++ .../engine_client/api_server/api_server.py | 75 +++++++++++++++---- .../api_server/container/container.py | 7 -- dockyard/engine_client/container/__init__.py | 0 dockyard/engine_client/container/container.py | 32 ++++++++ .../engine_client/container/post_container.py | 8 ++ .../engine_client/container/pre_container.py | 3 + dockyard/engine_client/image/__init__.py | 0 dockyard/engine_client/image/image.py | 3 + dockyard/engine_client/network/__init__.py | 0 dockyard/engine_client/network/network.py | 3 + .../engine_client/synchronizer/__init__.py | 25 +++++++ .../synchronizer}/synchronizer.py | 16 ++-- dockyard/engine_client/volume/__init__.py | 0 dockyard/engine_client/volume/volume.py | 3 + etc/dockyard/dockyard.conf | 7 ++ 24 files changed, 190 insertions(+), 82 deletions(-) rename dockyard/{engine/common/containers/store => common/stores}/__init__.py (100%) rename dockyard/{engine_client/api_server/container => common/stores/consul}/__init__.py (100%) rename dockyard/{engine/common/containers/store => common/stores}/consul/consul_client.py (65%) delete mode 100644 dockyard/engine/common/containers/store/consul/__init__.py delete mode 100644 dockyard/engine_client/api_server/container/container.py create mode 100644 dockyard/engine_client/container/__init__.py create mode 100644 dockyard/engine_client/container/container.py create mode 100644 dockyard/engine_client/container/post_container.py create mode 100644 dockyard/engine_client/container/pre_container.py create mode 100644 dockyard/engine_client/image/__init__.py create mode 100644 dockyard/engine_client/image/image.py create mode 100644 dockyard/engine_client/network/__init__.py create mode 100644 dockyard/engine_client/network/network.py create mode 100644 dockyard/engine_client/synchronizer/__init__.py rename dockyard/{engine/common/containers/store/consul => engine_client/synchronizer}/synchronizer.py (79%) create mode 100644 dockyard/engine_client/volume/__init__.py create mode 100644 dockyard/engine_client/volume/volume.py diff --git a/dockyard/common/base.py b/dockyard/common/base.py index 9660e2a..c80dc0e 100644 --- a/dockyard/common/base.py +++ b/dockyard/common/base.py @@ -62,17 +62,16 @@ def send(self, method, url, headers=None, post_params=None, :post_params: post parameters. :body: Request body. """ - #req_type = self._is_local_request() + req_type = self._is_local_request() # Pre processing needs to be done for dockyard feature before # performing some actions - # print method, url, headers, body - #if req_type: - # self.engine_client.pre_process(method=method, url=url, - # body=body, headers=headers) - #else: - # headers = self._add_headers(headers) + if req_type: + self.engine_client.process(method=method, url=url, r_type="pre", + body=body, headers=headers) + data = self.pool.urlopen(method, url, headers=headers, body=body).data # Post processing needs to be done for some of the dockyard operations - #self.engine_client.post_process(method=method, url=url, - # body=body, headers=headers) + if req_type: + self.engine_client.process(method=method, url=url, r_type="post", + body=body, headers=headers) return data diff --git a/dockyard/engine/common/containers/store/__init__.py b/dockyard/common/stores/__init__.py similarity index 100% rename from dockyard/engine/common/containers/store/__init__.py rename to dockyard/common/stores/__init__.py diff --git a/dockyard/engine_client/api_server/container/__init__.py b/dockyard/common/stores/consul/__init__.py similarity index 100% rename from dockyard/engine_client/api_server/container/__init__.py rename to dockyard/common/stores/consul/__init__.py diff --git a/dockyard/engine/common/containers/store/consul/consul_client.py b/dockyard/common/stores/consul/consul_client.py similarity index 65% rename from dockyard/engine/common/containers/store/consul/consul_client.py rename to dockyard/common/stores/consul/consul_client.py index a97d01e..6f9e309 100644 --- a/dockyard/engine/common/containers/store/consul/consul_client.py +++ b/dockyard/common/stores/consul/consul_client.py @@ -5,7 +5,7 @@ def __init__(self): self.consul = consul.Consul() def get(self, key): - self.consul.kv.get(key) + return self.consul.kv.get(key) def put(self, key, value): - self.consul.kv.put(key, value) + return self.consul.kv.put(key, value) diff --git a/dockyard/common/utils.py b/dockyard/common/utils.py index ca286d0..e8f6729 100644 --- a/dockyard/common/utils.py +++ b/dockyard/common/utils.py @@ -118,7 +118,6 @@ def is_localhost(host): for ip in localhost_ips: if ip == host['host']: return True - print ip return False diff --git a/dockyard/controllers/v1/container.py b/dockyard/controllers/v1/container.py index 8031041..0eaa9e6 100644 --- a/dockyard/controllers/v1/container.py +++ b/dockyard/controllers/v1/container.py @@ -15,7 +15,7 @@ def _call_operation(self, name_or_id, operation, **kwargs): abort(404) @expose() - def get(self, name_or_id, operation, **kwargs): + def get(self, name_or_id, operation=None, **kwargs): return self._call_operation(name_or_id, operation, **kwargs) @expose() diff --git a/dockyard/engine/common/containers/store/consul/__init__.py b/dockyard/engine/common/containers/store/consul/__init__.py deleted file mode 100644 index 6de8f25..0000000 --- a/dockyard/engine/common/containers/store/consul/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -from oslo_config import cfg -from threading import Thread -import time - -from dockyard.engine.common.containers.store.consul.synchronizer import ( - ContainerSynchronizer) - -CONSUL_DATABASE_OPT = [ - cfg.IntOpt('synchronization_time', - default=20, - help='Time after which data will be synced .'), -] - -CONF = cfg.CONF -opt_group = cfg.OptGroup(name='consul_database', - title='Group for the configuration options ' - 'for consul as database.') -CONF.register_group(opt_group) -CONF.register_opts(CONSUL_DATABASE_OPT, opt_group) - - -class Synchronizer(Thread): - def __init__(self): - Thread.__init__(self) - self.container = ContainerSynchronizer() - self.sleep_time = CONF.consul_database.synchronization_time - - - def run(self): - """This thread is responsible for synchronizations of containers, - and other docker resources. - """ - while True: - self.container.synchronize() - time.sleep(self.sleep_time) - - -sync = Synchronizer() -sync.setName("Synchronizer") -sync.start() diff --git a/dockyard/engine/controllers/__init__.py b/dockyard/engine/controllers/__init__.py index 1321a4b..692f9ea 100644 --- a/dockyard/engine/controllers/__init__.py +++ b/dockyard/engine/controllers/__init__.py @@ -1,7 +1,5 @@ from oslo_config import cfg -import dockyard.engine.common.containers.store.consul - NETWORK_DRIVER_OPT = [ cfg.StrOpt('network_driver', default='bridges.linux.LinuxBridgeManager', diff --git a/dockyard/engine_client/__init__.py b/dockyard/engine_client/__init__.py index e69de29..162fbdd 100644 --- a/dockyard/engine_client/__init__.py +++ b/dockyard/engine_client/__init__.py @@ -0,0 +1,24 @@ +import importlib +from oslo_config import cfg + +# Add default values for the database to be used for dockyard. +DATABASE_OPT = [ + cfg.StrOpt("driver", + default="consul.consul_client.ConsulKV", + help="Database driver for dockyard"), + cfg.IntOpt('synchronization_time', + default=20, + help='Time after which data will be synced .') +] + +CONF = cfg.CONF +opt_group = cfg.OptGroup(name='database', + title='Group for database') +CONF.register_group(opt_group) +CONF.register_opts(DATABASE_OPT, opt_group) +db_driver_loc = 'dockyard.common.stores' +db_driver_info = (('%s.%s') % (db_driver_loc, CONF.database.driver)) +module_name, db = db_driver_info.rsplit(".", 1) +module = importlib.import_module(module_name) + +#import synchronizer diff --git a/dockyard/engine_client/api_server/api_server.py b/dockyard/engine_client/api_server/api_server.py index 8fbd9f6..84e97fe 100644 --- a/dockyard/engine_client/api_server/api_server.py +++ b/dockyard/engine_client/api_server/api_server.py @@ -1,9 +1,64 @@ # This client will send request to api server, it sends request to API server from dockyard.engine_client.base import EngineClient -from dockyard.engine_client.api_server.container.container import ( - PreProcessor, - PostProcessor) +from dockyard.engine_client.container import container +from dockyard.engine_client.image import image +from dockyard.engine_client.network import network +from dockyard.engine_client.volume import volume + +class ProcessorRouter(object): + def __init__(self): + self.container = container.ContainerRouter() + self.image = image.Image() + self.network = network.Network() + self.volume = volume.Volume() + + def _get_module(self, url): + modules = url.rsplit(':', 1)[-1].split('/') + + if len(url.rsplit()[-1].split()) == 3: + module = modules[3] + else: + module = modules[2] + + return module + + def _call_operation(self, obj, operation, url, **kwargs): + try: + return getattr(obj, "process")(url, operation=operation, **kwargs) + except AttributeError: + # Currently no preprocessor or post processor are being done + # therefor it is passed otherwise InvalidOperation Exception + # has to be raised + pass + + def containers(self, url, **kwargs): + """This method calls appropriate method of the for the + preprocessing task. + """ + operation = self._get_module(url) + return self._call_operation(self.container, operation, url, **kwargs) + + def images(self, url, **kwargs): + """This method calls appropriate method for pre processing of + images. + """ + operation = self._get_module(url) + return self._call_operation(self.image, operation, url, **kwargs) + + def networks(self, url, **kwargs): + """This method is responsible for calling method of prepocessing + of networks. + """ + operation = self._get_module(url) + return self._call_operation(self.network, operation, url, **kwargs) + + def volumes(self, url, **kwargs): + """This method is routes preprocessing tasks to the appropriate + method of a class for volumes related operations. + """ + operation = self._get_module(url) + return self._call_operation(self.volume, operation, url, **kwargs) class APIServerEngineClient(EngineClient): @@ -12,17 +67,11 @@ class APIServerEngineClient(EngineClient): request and then perform operation. """ def __init__(self): - self.pre_process = PreProcessor() - self.post_process = PostProcessor() + self.router = ProcessorRouter() def _get_module(self, url): return url.rsplit(':', 1)[-1].split('/')[1] - def pre_process(self, url, **kwargs): - module = self._get_module(url, **kwargs) - return getattr(self.pre_process, module)(url, **kwargs) - - - def post_process(self, url, **kwargs): - module = self._get_module(url, **kwargs) - return getattr(self.post_process, module)(url, **kwargs) + def process(self, url, **kwargs): + module = self._get_module(url) + return getattr(self.router, module)(url, **kwargs) diff --git a/dockyard/engine_client/api_server/container/container.py b/dockyard/engine_client/api_server/container/container.py deleted file mode 100644 index 2a3918f..0000000 --- a/dockyard/engine_client/api_server/container/container.py +++ /dev/null @@ -1,7 +0,0 @@ -class PreProcessor(object): - def __ini__(self): - pass - -class PostProcessor(object): - def __init__(self): - pass diff --git a/dockyard/engine_client/container/__init__.py b/dockyard/engine_client/container/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine_client/container/container.py b/dockyard/engine_client/container/container.py new file mode 100644 index 0000000..ca13dde --- /dev/null +++ b/dockyard/engine_client/container/container.py @@ -0,0 +1,32 @@ +from dockyard.engine_client.container.pre_container import PreProcessor +from dockyard.engine_client.container.post_container import PostProcessor + +class ContainerRouter(object): + """This class takes care of post processing and pre processing + required for providing the functionality of dockyard. + """ + mapping = { + "pre": "pre_processor", + "post": "post_processor" + } + + def __init__(self): + self.pre_processor = PreProcessor() + self.post_processor = PostProcessor() + + def _call_operation(self, url, **kwargs): + try: + obj = getattr(self, ContainerRouter.mapping[kwargs["r_type"]]) + return getattr(obj, kwargs["operation"])(url, **kwargs) + except AttributeError: + # Currently no preprocessor or post processor are being done + # therefor it is passed otherwise InvalidOperation Exception + # has to be raised + pass + + def process(self, url, **kwargs): + """This method routes the request to appropriate method of the class + depending on the whether it is pre processing request or post + processing request. + """ + self._call_operation(url=url, **kwargs) diff --git a/dockyard/engine_client/container/post_container.py b/dockyard/engine_client/container/post_container.py new file mode 100644 index 0000000..8622b69 --- /dev/null +++ b/dockyard/engine_client/container/post_container.py @@ -0,0 +1,8 @@ +class PostProcessor(object): + def __init__(self): + pass + + def create(self): + """This method does the pre processing required for dockyard functionality. + """ + diff --git a/dockyard/engine_client/container/pre_container.py b/dockyard/engine_client/container/pre_container.py new file mode 100644 index 0000000..4a9cedc --- /dev/null +++ b/dockyard/engine_client/container/pre_container.py @@ -0,0 +1,3 @@ +class PreProcessor(object): + def __init__(self): + pass diff --git a/dockyard/engine_client/image/__init__.py b/dockyard/engine_client/image/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine_client/image/image.py b/dockyard/engine_client/image/image.py new file mode 100644 index 0000000..021e2a9 --- /dev/null +++ b/dockyard/engine_client/image/image.py @@ -0,0 +1,3 @@ +class Image(object): + def __init__(self): + pass diff --git a/dockyard/engine_client/network/__init__.py b/dockyard/engine_client/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine_client/network/network.py b/dockyard/engine_client/network/network.py new file mode 100644 index 0000000..0f19b9f --- /dev/null +++ b/dockyard/engine_client/network/network.py @@ -0,0 +1,3 @@ +class Network(object): + def __init__(self): + pass diff --git a/dockyard/engine_client/synchronizer/__init__.py b/dockyard/engine_client/synchronizer/__init__.py new file mode 100644 index 0000000..b052176 --- /dev/null +++ b/dockyard/engine_client/synchronizer/__init__.py @@ -0,0 +1,25 @@ +from oslo_config import cfg +from threading import Thread +import time + +from synchronizer import ContainerSynchronizer + +class Synchronizer(Thread): + def __init__(self): + Thread.__init__(self) + self.container = ContainerSynchronizer() + self.sleep_time = CONF.database.synchronization_time + + + def run(self): + """This thread is responsible for synchronizations of containers, + and other docker resources. + """ + while True: + self.container.synchronize() + time.sleep(self.sleep_time) + + +sync = Synchronizer() +sync.setName("Synchronizer") +sync.start() diff --git a/dockyard/engine/common/containers/store/consul/synchronizer.py b/dockyard/engine_client/synchronizer/synchronizer.py similarity index 79% rename from dockyard/engine/common/containers/store/consul/synchronizer.py rename to dockyard/engine_client/synchronizer/synchronizer.py index 44ecff2..4c02f78 100644 --- a/dockyard/engine/common/containers/store/consul/synchronizer.py +++ b/dockyard/engine_client/synchronizer/synchronizer.py @@ -1,14 +1,13 @@ import ast from oslo_config import cfg -from dockyard.common.utils import get_localhost_ip +#from dockyard.common.utils import get_localhost_ip from dockyard.common.container.container import Container -from dockyard.engine.common.containers.store.consul import consul_client +from dockyard.engine_client import module, db CONF = cfg.CONF - class ContainerSynchronizer(object): def __init__(self): if CONF.docker.docker_host == "0.0.0.0": @@ -21,13 +20,15 @@ def __init__(self): port = CONF.docker.docker_port self.host = { 'host': host, 'port': port } self.container = Container() - self.db = consul_client.ConsulKV() + self.db = getattr(module, db)() def _containers(self): """This method fetch all the containers running on local machines. """ containers = self.container.list(host=self.host) + containers = containers.replace("null", '"null"') containers = ast.literal_eval(containers) + for con in containers: yield con @@ -47,6 +48,7 @@ def _register(self, container): """This method is responsible for registering a container. """ id_ = container["Id"] - container_info = self._inspect(id_=id_) - key = str((id_, self.host)) - self.db.put(key, container_info) + container_info = {"host": self.host, + "container_info": self._inspect(id_=id_)} + + self.db.put(id_, str(container_info)) diff --git a/dockyard/engine_client/volume/__init__.py b/dockyard/engine_client/volume/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dockyard/engine_client/volume/volume.py b/dockyard/engine_client/volume/volume.py new file mode 100644 index 0000000..d17ea02 --- /dev/null +++ b/dockyard/engine_client/volume/volume.py @@ -0,0 +1,3 @@ +class Volume(object): + def __init__(self): + pass diff --git a/etc/dockyard/dockyard.conf b/etc/dockyard/dockyard.conf index c1da802..2819807 100644 --- a/etc/dockyard/dockyard.conf +++ b/etc/dockyard/dockyard.conf @@ -41,7 +41,14 @@ docker_port = 2375 # service name docker_name = 'docker' +[database] +# Database driver to be used for storing data. +driver = consul.consul_client.ConsulKV + [consul_database] # This is the time after which synchronizer thread will wake up and start # syncing containers launched. synchronization_time = 20 + +#Secret used to put data into the consul and to retrieve from the database +secret = secret_information From 92ea0313ae58fab775edf0bcdc73057d569efffe Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Tue, 5 Jul 2016 22:20:16 +0530 Subject: [PATCH 59/62] Added support for synchronization --- dockyard/common/__init__.py | 79 +++++++++++++++++++ .../engine_client/api_server/api_server.py | 5 +- .../engine_client/container/post_container.py | 12 ++- .../engine_client/synchronizer/__init__.py | 25 ------ .../synchronizer/synchronizer.py | 42 ++-------- 5 files changed, 101 insertions(+), 62 deletions(-) diff --git a/dockyard/common/__init__.py b/dockyard/common/__init__.py index e69de29..7cbc5ab 100644 --- a/dockyard/common/__init__.py +++ b/dockyard/common/__init__.py @@ -0,0 +1,79 @@ +import ast +import importlib +from oslo_config import cfg +from threading import Thread +import time + +from dockyard.common.container.container import Container +from dockyard.common.utils import get_localhost_ip +from dockyard.engine_client.synchronizer.synchronizer import ( + ContainerSynchronizer) + +CONF = cfg.CONF + + +class Synchronizer(Thread): + def __init__(self): + if CONF.docker.docker_host == "0.0.0.0": + for ip in get_localhost_ip(): + host = ip + break + else: + host = CONF.docker.docker_host + + port = CONF.docker.docker_port + self.host = { 'host': host, 'port': port } + Thread.__init__(self) + self.container = Container() + self.synchronizer = ContainerSynchronizer() + self.sleep_time = CONF.database.synchronization_time + + + def run(self): + """This thread is responsible for synchronizations of containers, + and other docker resources. + """ + while True: + self._synchronize() + time.sleep(self.sleep_time) + + def _get_format(self, container): + """This method converts a container information into the required format + for the container. + """ + container_info = {"host": self.host, "Container-Info": container} + key = (self.host, container["Id"], "container") + return (key, container_info) + + + def _synchronize(self): + """This method fetch all the containers running on local machines. + """ + containers = self.container.list(host=self.host) + containers = containers.replace("null", '"null"') + containers = ast.literal_eval(containers) + + for con in containers: + container_info = self._get_format(self._inspect(con)) + self.synchronizer.synchronize([container_info]) + + + def _inspect(self, container): + container = self.container.list(name_or_id=container["Id"], + host=self.host) + + container = container.replace("null", '"null"') + container = container.replace("false", '"False"') + container = container.replace("true", '"True"') + return ast.literal_eval(container) + + +def run_synchronizer(): + """This method is responsible for synchronizing containers information + with the consul databse. + """ + sync = Synchronizer() + sync.setName("Synchronizer") + sync.start() + +run_synchronizer() diff --git a/dockyard/engine_client/api_server/api_server.py b/dockyard/engine_client/api_server/api_server.py index 84e97fe..3e60c69 100644 --- a/dockyard/engine_client/api_server/api_server.py +++ b/dockyard/engine_client/api_server/api_server.py @@ -74,4 +74,7 @@ def _get_module(self, url): def process(self, url, **kwargs): module = self._get_module(url) - return getattr(self.router, module)(url, **kwargs) + try: + return getattr(self.router, module)(url, **kwargs) + except: + return diff --git a/dockyard/engine_client/container/post_container.py b/dockyard/engine_client/container/post_container.py index 8622b69..f3573a7 100644 --- a/dockyard/engine_client/container/post_container.py +++ b/dockyard/engine_client/container/post_container.py @@ -1,8 +1,16 @@ +from dockyard.common.stores.consul.consul_client import ConsulKV + + class PostProcessor(object): def __init__(self): - pass + self.db = ConsulKV() - def create(self): + def create(self, url, **kwargs): """This method does the pre processing required for dockyard functionality. """ + def list(self, url, **kwargs): + """This method is list all the containers on different hosts + as well as one container. + """ + self.db diff --git a/dockyard/engine_client/synchronizer/__init__.py b/dockyard/engine_client/synchronizer/__init__.py index b052176..e69de29 100644 --- a/dockyard/engine_client/synchronizer/__init__.py +++ b/dockyard/engine_client/synchronizer/__init__.py @@ -1,25 +0,0 @@ -from oslo_config import cfg -from threading import Thread -import time - -from synchronizer import ContainerSynchronizer - -class Synchronizer(Thread): - def __init__(self): - Thread.__init__(self) - self.container = ContainerSynchronizer() - self.sleep_time = CONF.database.synchronization_time - - - def run(self): - """This thread is responsible for synchronizations of containers, - and other docker resources. - """ - while True: - self.container.synchronize() - time.sleep(self.sleep_time) - - -sync = Synchronizer() -sync.setName("Synchronizer") -sync.start() diff --git a/dockyard/engine_client/synchronizer/synchronizer.py b/dockyard/engine_client/synchronizer/synchronizer.py index 4c02f78..b4e8e48 100644 --- a/dockyard/engine_client/synchronizer/synchronizer.py +++ b/dockyard/engine_client/synchronizer/synchronizer.py @@ -1,54 +1,28 @@ import ast +import netifaces from oslo_config import cfg -#from dockyard.common.utils import get_localhost_ip -from dockyard.common.container.container import Container from dockyard.engine_client import module, db - CONF = cfg.CONF + class ContainerSynchronizer(object): def __init__(self): - if CONF.docker.docker_host == "0.0.0.0": - for ip in get_localhost_ip(): - host = ip - break - else: - host = CONF.docker.docker_host - - port = CONF.docker.docker_port - self.host = { 'host': host, 'port': port } - self.container = Container() self.db = getattr(module, db)() - def _containers(self): - """This method fetch all the containers running on local machines. - """ - containers = self.container.list(host=self.host) - containers = containers.replace("null", '"null"') - containers = ast.literal_eval(containers) - - for con in containers: - yield con - - def synchronize(self): + def synchronize(self, containers_info): """This method is responsible for initilizing databases at the start of dockyard. It collects all the containers running on localhost and putt the data in the consul """ - for container in self._containers(): + for container in containers_info: self._register(container) - def _inspect(self, id_): - return self.container.list(name_or_id=id_, host=self.host) - - def _register(self, container): + def _register(self, container_info): """This method is responsible for registering a container. """ - id_ = container["Id"] - container_info = {"host": self.host, - "container_info": self._inspect(id_=id_)} - - self.db.put(id_, str(container_info)) + (key, container) = container_info + print container_info + return self.db.put(str(key), str(container)) From 800e5d97ef4da8d2d8251647b8fa5d4eeccb68d2 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Tue, 5 Jul 2016 23:11:34 +0530 Subject: [PATCH 60/62] Added synchronization for all objects --- dockyard/common/__init__.py | 62 ++++++++++++------- dockyard/common/image/image.py | 27 ++++---- dockyard/common/network/network.py | 20 +++--- dockyard/common/volume/volume.py | 12 ++-- .../synchronizer/synchronizer.py | 13 ++-- 5 files changed, 76 insertions(+), 58 deletions(-) diff --git a/dockyard/common/__init__.py b/dockyard/common/__init__.py index 7cbc5ab..4d91ebd 100644 --- a/dockyard/common/__init__.py +++ b/dockyard/common/__init__.py @@ -2,9 +2,11 @@ import importlib from oslo_config import cfg from threading import Thread -import time - +import time from dockyard.common.container.container import Container +from dockyard.common.image.image import Image +from dockyard.common.network.network import Network +from dockyard.common.volume.volume import Volume from dockyard.common.utils import get_localhost_ip from dockyard.engine_client.synchronizer.synchronizer import ( ContainerSynchronizer) @@ -25,6 +27,9 @@ def __init__(self): self.host = { 'host': host, 'port': port } Thread.__init__(self) self.container = Container() + self.image = Image() + self.network = Network() + self.volume = Volume() self.synchronizer = ContainerSynchronizer() self.sleep_time = CONF.database.synchronization_time @@ -37,37 +42,48 @@ def run(self): self._synchronize() time.sleep(self.sleep_time) - def _get_format(self, container): + def _get_format(self, id_, value, type_): """This method converts a container information into the required format for the container. """ - container_info = {"host": self.host, "Container-Info": container} - key = (self.host, container["Id"], "container") - return (key, container_info) + value = {"host": self.host, type_: value} + key = (self.host["host"], id_, type_) + return (key, value) + + def _synchronize_container(self): + containers = self.container.list(host=self.host) + return self._sync(containers, type_="container") + + def _synchronize_image(self): + images = self.image.list(host=self.host) + return self._sync(images, type_="image") + + def _synchronize_network(self): + networks = self.network.list(host=self.host) + return self._sync(networks, type_="network") + + def _synchronize_volume(self): + volumes = self.container.list(host=self.host) + return self._sync(volumes, type_="volume") + def _sync(self, info_s, type_=None): + info_s = info_s.replace("null", "None") + info_s = info_s.replace("true", "True") + info_s = info_s.replace("false", "False") + info_s = ast.literal_eval(info_s) + for info in info_s: + info = self._get_format(info["Id"], info, type_) + self.synchronizer.synchronize([info]) def _synchronize(self): """This method fetch all the containers running on local machines. """ - containers = self.container.list(host=self.host) - containers = containers.replace("null", '"null"') - containers = ast.literal_eval(containers) - - for con in containers: - container_info = self._get_format(self._inspect(con)) - self.synchronizer.synchronize([container_info]) + self._synchronize_container() + self._synchronize_volume() + self._synchronize_image() + self._synchronize_network() - def _inspect(self, container): - container = self.container.list(name_or_id=container["Id"], - host=self.host) - - container = container.replace("null", '"null"') - container = container.replace("false", '"False"') - container = container.replace("true", '"True"') - return ast.literal_eval(container) - - def run_synchronizer(): """This method is responsible for synchronizing containers information with the consul databse. diff --git a/dockyard/common/image/image.py b/dockyard/common/image/image.py index 523ddd3..e1bcb17 100644 --- a/dockyard/common/image/image.py +++ b/dockyard/common/image/image.py @@ -9,31 +9,34 @@ class Image(object): def __init__(self): self.url = url.URL(self.base_url) - def list(self, id_=None): + def list(self, id_=None, host=None): url_ = self.url.make_url(url_='json', id_=id_) - return utils.dispatch_get_request(url=url_) + return utils.dispatch_get_request(url=url_, host=host) - def history(self, id_=None): + def history(self, id_=None, host=None): url_ = self.url.make_url(url_='history', id_=id_) - return utils.dispatch_get_request(url=url_) + return utils.dispatch_get_request(url=url_, host=host) - def search(self, term=None): + def search(self, term=None, host=None): url_ = self.url.make_url(url_='search') query = {"term": term} - return utils.dispatch_get_request(url=url_, query_params=query) + return utils.dispatch_get_request(url=url_, query_params=query, + host=host) - def create(self, fromImage, tag): + def create(self, fromImage, tag, host=None): url_ = self.url.make_url(url_='create') query = {"fromImage": fromImage, "tag": tag} - return utils.dispatch_post_request(url=url_, query_params=query) + return utils.dispatch_post_request(url=url_, query_params=query, + host=host) def push(self, _id): abort(404) - def delete(self, id_): + def delete(self, id_, host=None): url_ = self.url.make_url(id_=id_) - return utils.dispatch_delete_request(url=url_) + return utils.dispatch_delete_request(url=url_, host=host) - def tag(self, _id=None, **kwargs): + def tag(self, _id=None, host=None, **kwargs): url_ = self.url.make_url(url_='tag', id_=id_) - return utils.dispatch_post_request(url=url_, query_params=kwargs) + return utils.dispatch_post_request(url=url_, query_params=kwargs, + host=host) diff --git a/dockyard/common/network/network.py b/dockyard/common/network/network.py index ed5089d..2139bd2 100644 --- a/dockyard/common/network/network.py +++ b/dockyard/common/network/network.py @@ -10,28 +10,28 @@ class DockerNetwork(object): def __init__(self): pass - def list(self, name_or_id=None): + def list(self, name_or_id=None, host=None): url_ = self.url.make_url(id_=name_or_id) - return utils.dispatch_get_request(url=url_) + return utils.dispatch_get_request(url=url_, host=host) - def connect(self, id_, **kwargs): + def connect(self, id_, host=None, **kwargs): body = request.body url_ = self.url.make_url(url_='connect', id_=id_) - return utils.dispatch_post_request(url=url_, body=body) + return utils.dispatch_post_request(url=url_, body=body, host=host) - def disconnect(self, id_, **kwargs): + def disconnect(self, id_, host=None, **kwargs): url_ = self.url.make_url(url_='disconnect', id_=id_) body = request.body - return utils.dispatch_post_request(url=url_, body=body) + return utils.dispatch_post_request(url=url_, body=body, host=host) - def create(self, **kwargs): + def create(self, host=None, **kwargs): url_ = self.url.make_url(url_='create') body = request.body - return utils.dispatch_post_request(url=url_, body=body) + return utils.dispatch_post_request(url=url_, body=body, host=host) - def delete(self, id_): + def delete(self, id_, host=None): url_ = self.url.make_url(id_=id_) - return utils.dispatch_delete_request(url=url_) + return utils.dispatch_delete_request(url=url_, host=host) class DockyardNetwork(object): diff --git a/dockyard/common/volume/volume.py b/dockyard/common/volume/volume.py index 8efed72..6820e85 100644 --- a/dockyard/common/volume/volume.py +++ b/dockyard/common/volume/volume.py @@ -9,14 +9,14 @@ class Volume(object): def __init__(self): self.url = url.URL(self.base_url) - def list(self, name=None): + def list(self, name=None, host=None): url_ = self.url.make_url(id_=name) - return utils.dispatch_get_request(url_) + return utils.dispatch_get_request(url_, host=host) - def delete(self, name): + def delete(self, name, host=None): url_ = self.url.make_url(id_=name) - return utils.dispatch_delete_request(url_) + return utils.dispatch_delete_request(url_, host=host) - def create(self, data): + def create(self, data, host=None): url_ = self.url.make_url(url_='create') - return utils.dispatch_post_request(url=url_, body=data) + return utils.dispatch_post_request(url=url_, body=data, host=None) diff --git a/dockyard/engine_client/synchronizer/synchronizer.py b/dockyard/engine_client/synchronizer/synchronizer.py index b4e8e48..a99c985 100644 --- a/dockyard/engine_client/synchronizer/synchronizer.py +++ b/dockyard/engine_client/synchronizer/synchronizer.py @@ -12,17 +12,16 @@ def __init__(self): self.db = getattr(module, db)() - def synchronize(self, containers_info): + def synchronize(self, infos): """This method is responsible for initilizing databases at the start of dockyard. It collects all the containers running on localhost and putt the data in the consul """ - for container in containers_info: - self._register(container) + for info in infos: + self._register(info) - def _register(self, container_info): + def _register(self, info): """This method is responsible for registering a container. """ - (key, container) = container_info - print container_info - return self.db.put(str(key), str(container)) + (key, value) = info + return self.db.put(str(key), str(info)) From 9dfb7f139df7a1b086e57afdef81c89d98f1530a Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Wed, 6 Jul 2016 23:03:37 +0530 Subject: [PATCH 61/62] added change for storing only IDs of resources --- dockyard/common/__init__.py | 17 ++++++++++++----- .../engine_client/synchronizer/synchronizer.py | 12 +++++++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/dockyard/common/__init__.py b/dockyard/common/__init__.py index 4d91ebd..9a22b55 100644 --- a/dockyard/common/__init__.py +++ b/dockyard/common/__init__.py @@ -42,12 +42,12 @@ def run(self): self._synchronize() time.sleep(self.sleep_time) - def _get_format(self, id_, value, type_): + def _get_format(self, value, type_): """This method converts a container information into the required format for the container. """ value = {"host": self.host, type_: value} - key = (self.host["host"], id_, type_) + key = (self.host["host"], type_) return (key, value) def _synchronize_container(self): @@ -66,14 +66,21 @@ def _synchronize_volume(self): volumes = self.container.list(host=self.host) return self._sync(volumes, type_="volume") + def _get_ids(self, info_s): + """This method returns all the values ids from the list of + dictionary received. + """ + ids = [x["Id"] for x in info_s] + return ids + def _sync(self, info_s, type_=None): info_s = info_s.replace("null", "None") info_s = info_s.replace("true", "True") info_s = info_s.replace("false", "False") info_s = ast.literal_eval(info_s) - for info in info_s: - info = self._get_format(info["Id"], info, type_) - self.synchronizer.synchronize([info]) + info_s = self._get_ids(info_s) + info_s = self._get_format(value=info_s, type_=type_) + self.synchronizer.synchronize([info_s]) def _synchronize(self): """This method fetch all the containers running on local machines. diff --git a/dockyard/engine_client/synchronizer/synchronizer.py b/dockyard/engine_client/synchronizer/synchronizer.py index a99c985..1058ab3 100644 --- a/dockyard/engine_client/synchronizer/synchronizer.py +++ b/dockyard/engine_client/synchronizer/synchronizer.py @@ -18,10 +18,16 @@ def synchronize(self, infos): and putt the data in the consul """ for info in infos: - self._register(info) + self._sync(info) - def _register(self, info): + def _get_value(self, key): + """This method returns all the values corresponding to a + key and resturn. + """ + return self.db.get(key) + + def _sync(self, info): """This method is responsible for registering a container. """ (key, value) = info - return self.db.put(str(key), str(info)) + return self.db.put(str(key), str(value)) From cc1a0f9cedd982e61d27ce5d1dc972457a293ec3 Mon Sep 17 00:00:00 2001 From: Rajiv Kumar Date: Wed, 6 Jul 2016 23:10:43 +0530 Subject: [PATCH 62/62] To optimize speed of earch converted list to dict --- dockyard/common/__init__.py | 6 +++++- dockyard/engine_client/synchronizer/synchronizer.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dockyard/common/__init__.py b/dockyard/common/__init__.py index 9a22b55..46ae25d 100644 --- a/dockyard/common/__init__.py +++ b/dockyard/common/__init__.py @@ -46,7 +46,11 @@ def _get_format(self, value, type_): """This method converts a container information into the required format for the container. """ - value = {"host": self.host, type_: value} + f_val = dict() + for v in value: + f_val[v] = "OK" + + value = {type_: f_val} key = (self.host["host"], type_) return (key, value) diff --git a/dockyard/engine_client/synchronizer/synchronizer.py b/dockyard/engine_client/synchronizer/synchronizer.py index 1058ab3..0b5ce2f 100644 --- a/dockyard/engine_client/synchronizer/synchronizer.py +++ b/dockyard/engine_client/synchronizer/synchronizer.py @@ -30,4 +30,5 @@ def _sync(self, info): """This method is responsible for registering a container. """ (key, value) = info + print info return self.db.put(str(key), str(value))