Big GLUE 2.1 rebase (#99)

* Fix world
* Add GLUE2.1 output

Jenkinsfile

@@ -88,7 +88,7 @@ pipeline {
                         sh 'mkdir -p ~/rpmbuild/{SOURCES,SPECS}'
                         sh 'cp dist/cloud_info_provider*.tar.gz ~/rpmbuild/SOURCES/'
                         sh 'cp rpm/cloud-info-provider*.spec ~/rpmbuild/SPECS/'
-                        //sh 'rpmbuild -ba ~/rpmbuild/SPECS/cloud-info-provider.spec'
+                        sh 'rpmbuild -ba ~/rpmbuild/SPECS/cloud-info-provider.spec'
                         sh 'rpmbuild -ba ~/rpmbuild/SPECS/cloud-info-provider-openstack.spec'
                         sh 'rpmbuild -ba ~/rpmbuild/SPECS/cloud-info-provider-opennebula.spec'
                         sh 'cp ~/rpmbuild/SRPMS/*.rpm ~/rpmbuild/RPMS/noarch/*.rpm ${WORKSPACE}/rpm/'

cloud_info/core.py → cloud_info_provider/core.py

@@ -1,18 +1,20 @@
 import argparse
 import os.path
 
-from cloud_info import exceptions
-from cloud_info import importutils
+from cloud_info_provider import exceptions
+from cloud_info_provider import importutils
 
+import mako.exceptions
 import mako.template
 
 SUPPORTED_MIDDLEWARE = {
-    'openstack': 'cloud_info.providers.openstack.OpenStackProvider',
-    'opennebula': 'cloud_info.providers.opennebula.OpenNebulaProvider',
-    'indigoon': 'cloud_info.providers.opennebula.IndigoONProvider',
-    'opennebularocci': 'cloud_info.providers.opennebula.'
+    'openstack': 'cloud_info_provider.providers.openstack.OpenStackProvider',
+    'opennebula': 'cloud_info_provider.providers.opennebula.'
+                  'OpenNebulaProvider',
+    'indigoon': 'cloud_info_provider.providers.opennebula.IndigoONProvider',
+    'opennebularocci': 'cloud_info_provider.providers.opennebula.'
                        'OpenNebulaROCCIProvider',
-    'static': 'cloud_info.providers.static.StaticProvider',
+    'static': 'cloud_info_provider.providers.static.StaticProvider',
 }
 
 
@@ -45,26 +47,12 @@ def load_templates(self):
                                          '%s.%s' % (tpl, template_extension))
             self.templates_files[tpl] = template_file
 
-    def _get_info_from_providers(self, method, provider_opts=None):
-        # XXX Temporarily update dynamic provider parameters
-        # XXX Required to be able to pass a custom project to the provider
-        # XXX to retrieve project-specific templates and images
-        if provider_opts:
-            opts = self.opts
-            d = vars(opts)
-            for k, v in provider_opts.items():
-                d[k] = v
-
-            provider_cls = importutils.import_class(
-                SUPPORTED_MIDDLEWARE[opts.middleware]
-            )
-            self.dynamic_provider = provider_cls(opts)
-
+    def _get_info_from_providers(self, method, **provider_kwargs):
         info = {}
         for i in (self.static_provider, self.dynamic_provider):
             if not i:
                 continue
-            result = getattr(i, method)()
+            result = getattr(i, method)(**provider_kwargs)
             info.update(result)
         return info
 
@@ -73,7 +61,10 @@ def _format_template(self, template, info, extra={}):
         info.update(extra)
         t = self.templates_files[template]
         tpl = mako.template.Template(filename=t)
-        return tpl.render(attributes=info)
+        try:
+            return tpl.render(attributes=info)
+        except Exception:
+            return mako.exceptions.text_error_template().render()
 
 
 class StorageBDII(BaseBDII):
@@ -109,32 +100,68 @@ def __init__(self, opts):
         self.templates = ['compute']
 
     def render(self):
-        endpoints = self._get_info_from_providers('get_compute_endpoints')
-
-        if not endpoints.get('endpoints'):
-            return ''
+        info = {}
 
+        # Retrieve global site information
+        # XXX Validate if really project agnostic
+        # XXX Here it uses the "default" project from the CLI parameters
         site_info = self._get_info_from_providers('get_site_info')
-        static_compute_info = dict(endpoints, **site_info)
-        static_compute_info.pop('endpoints')
 
-        templates = self._get_info_from_providers('get_templates')
-        images = self._get_info_from_providers('get_images')
+        # Get shares / projects and related images and templates
+        shares = self._get_info_from_providers('get_compute_shares')
 
-        for url, endpoint in endpoints['endpoints'].items():
-            endpoint.update(static_compute_info)
+        for share in shares.values():
+            kwargs = share.copy()
 
-        for template_id, template in templates.items():
-            template.update(static_compute_info)
+            endpoints = self._get_info_from_providers('get_compute_endpoints',
+                                                      **kwargs)
 
-        for image_id, image in images.items():
-            image.update(static_compute_info)
+            if not endpoints.get('endpoints'):
+                return ''
+
+            # Collect static information for endpoints
+            static_compute_info = dict(endpoints, **site_info)
+            static_compute_info.pop('endpoints')
+
+            # Add same static information to all endpoints
+            for endpoint in endpoints['endpoints'].values():
+                endpoint.update(static_compute_info)
+
+            images = self._get_info_from_providers('get_images',
+                                                   **kwargs)
+
+            templates = self._get_info_from_providers('get_templates',
+                                                      **kwargs)
+
+            instances = self._get_info_from_providers('get_instances',
+                                                      **kwargs)
+
+            quotas = self._get_info_from_providers('get_compute_quotas',
+                                                   **kwargs)
+
+            for template in templates.values():
+                template.update(static_compute_info)
+
+            for image in images.values():
+                image.update(static_compute_info)
+
+            share['images'] = images
+            share['templates'] = templates
+            share['instances'] = instances
+            share['endpoints'] = endpoints
+            share['quotas'] = quotas
+
+        # XXX Avoid creating a new list
+        endpoints = {endpoint_id: endpoint for share_id, share in
+                     shares.items() for endpoint_id,
+                     endpoint in share['endpoints'].items()}
+
+        # XXX Avoid redoing what was done in the previous shares loop
+        static_compute_info = dict(endpoints, **site_info)
+        static_compute_info.pop('endpoints')
 
-        info = {}
-        info.update({'endpoints': endpoints})
         info.update({'static_compute_info': static_compute_info})
-        info.update({'templates': templates})
-        info.update({'images': images})
+        info.update({'shares': shares})
 
         return self._format_template('compute', info)
 

cloud_info_provider/providers/__init__.py

@@ -0,0 +1,31 @@
+class BaseProvider(object):
+    def __init__(self, opts):
+        self.opts = opts
+
+    def get_site_info(self, **kwargs):
+        return {}
+
+    def get_images(self, **kwargs):
+        return {}
+
+    def get_templates(self, **kwargs):
+        return {}
+
+    def get_instances(self, **kwargs):
+        return {}
+
+    def get_compute_shares(self, **kwargs):
+        return {}
+
+    def get_compute_quotas(self, **kwargs):
+        return {}
+
+    def get_compute_endpoints(self, **kwargs):
+        return {}
+
+    def get_storage_endpoints(self, **kwargs):
+        return {}
+
+    @staticmethod
+    def populate_parser(parser):
+        '''Populate the argparser 'parser' with the needed options.'''

cloud_info/providers/opennebula.py → cloud_info_provider/providers/opennebula.py

@@ -2,9 +2,10 @@
 import os
 import string
 
-from cloud_info import exceptions
-from cloud_info import providers
-from cloud_info import utils
+from cloud_info_provider import exceptions
+from cloud_info_provider import providers
+from cloud_info_provider.providers import ssl_utils
+from cloud_info_provider import utils
 
 try:
     import defusedxml.ElementTree
@@ -74,7 +75,7 @@ def _get_one_documents(self, document_type):
             self.on_auth, -3, -1, -1, document_type)
         return self._handle_response(response)
 
-    def get_images(self):
+    def get_images(self, **kwargs):
         template = {
             'image_name': None,
             'image_description': None,
@@ -88,9 +89,13 @@ def get_images(self):
         }
         defaults = self.static.get_image_defaults(prefix=True)
         img_schema = defaults.get('image_schema', 'template')
+        group_name = kwargs.get('project', None)
 
         templates = {}
+        # TODO(enolfc): cache the templates instead of always calling this?
         for tpl_id, tpl in self._get_one_templates().items():
+            if group_name and group_name != tpl.get('gname'):
+                continue
             aux_tpl = template.copy()
             aux_tpl.update(defaults)
 
@@ -176,7 +181,7 @@ class IndigoONProvider(OpenNebulaBaseProvider):
     def __init__(self, opts):
         super(IndigoONProvider, self).__init__(opts)
 
-    def get_templates(self):
+    def get_templates(self, **kwargs):
         template = {
             'template_id': None,
             'template_name': None,
@@ -218,7 +223,7 @@ def get_templates(self):
             templates[tpl_id] = aux_tpl
         return templates
 
-    def get_images(self):
+    def get_images(self, **kwargs):
         image = {
             'image_name': None,
             'image_id': None,
@@ -266,9 +271,29 @@ def __init__(self, opts):
             msg = ('ERROR, You must provide a rocci_template_dir '
                    'via --rocci-template-dir')
             raise exceptions.OpenNebulaProviderException(msg)
+        self.ca_info = {}
         super(OpenNebulaROCCIProvider, self).__init__(opts)
 
-    def get_templates(self):
+    def _get_endpoint_ca_information(self, url, **kwargs):
+        if url not in self.ca_info:
+            ca_info = ssl_utils.get_endpoint_ca_information(url, **kwargs)
+            self.ca_info[url] = ca_info
+        return self.ca_info[url]
+
+    def get_compute_endpoints(self, **kwargs):
+        epts = dict()
+        static_epts = self.static.get_compute_endpoints(**kwargs)
+        for url, static_ept in static_epts['endpoints'].items():
+            ept = static_ept.copy()
+            ca_info = self._get_endpoint_ca_information(url)
+            ept.update({
+                'endpoint_trusted_cas': ca_info['trusted_cas'],
+                'endpoint_issuer': ca_info['issuer'],
+            })
+            epts[url] = ept
+        return {'endpoints': epts}
+
+    def get_templates(self, **kwargs):
         """Get flavors from rOCCI-server configuration."""
         template = {
             'template_id': None,
@@ -282,7 +307,8 @@ def get_templates(self):
         template.update(self.static.get_template_defaults(prefix=True))
 
         if self.rocci_remote_templates:
-            templates = self.remote_templates(template)
+            group_name = kwargs.get('project', None)
+            templates = self.remote_templates(template, group_name)
         else:
             templates = self.local_templates(template)
 
@@ -318,12 +344,15 @@ def local_templates(self, template):
 
         return templates
 
-    def remote_templates(self, template):
+    def remote_templates(self, template, group_name):
         document_type = 999  # TODO(bparak): configurable?
 
         templates = {}
+        # TODO(enolfc): cache info from ONE?
         for doc_id, doc in self._get_one_documents(document_type).items():
             document = json.loads(doc['template']['body'])
+            if group_name and group_name != doc.get('gname'):
+                continue
 
             aux = template.copy()
             aux.update({