diff --git a/INSTALL b/INSTALL index f0214bb..ff0eb4d 100644 --- a/INSTALL +++ b/INSTALL @@ -5,8 +5,8 @@ IM is based on python, so Python 2.4 or higher runtime and standard library must be installed in the system. -It is also required to install the Python Lex & Yacc library (http://www.dabeaz.com/ply/). -It is available in all of the main distributions as 'python-ply' package. +It is also required to install the RADL parser (https://github.com/grycap/radl), available in pip +as the 'RADL' package. 1.2 OPTIONAL PACKAGES @@ -38,48 +38,3 @@ xmlrpc_url=http://localhost:8899 auth_file=auth.dat xmlrpc_ssl=no xmlrpc_ssl_ca_certs=/tmp/pki/ca-chain.pem - -1.4.1 AUTH FILE - -The authorization data is used to validate access to the components in the -infrastructure. This file is composed of a set of "key - value" pairs, -where the user specifies the authorization data for all the components and cloud -deployments available. File auth.dat shows examples of authorization data. - -The list of "key" values that must be specified for each component are: - -* id: An optional field used to identify the virtual deployment. It must be unique - in the authorization data. -* type: The type of the component. It can be any of the components of the - architecture, such as the "InfrastructureManager", "VMRC" or any of - the cloud deployments currently supported by the IM: OpenNebula, EC2, - OpenStack, OCCI or LibVirt. -* username: The name of the user for the authentication. In the EC2 and OpenStack - cases it refers to the Access Key ID value. password: The password for - the authentication. In the EC2 and OpenStack cases it refers to the - Secret Access Key value. -* host: The address to the server in format "address:port" to specify the cloud - deployment to access. In the EC2 and in the system components (IM and VMRC) - this field is not used. - -An example of the auth file: - -id = one; type = OpenNebula; host = osenserve:2633; username = user; password = pass -type = InfrastructureManager; username = user; password = pass -type = VMRC; host = http://server:8080/vmrc; username = user; password = pass -id = ec2; type = EC2; username = ACCESS_KEY; password = SECRET_KEY -id = oshost; type = OpenStack; host = oshost:8773; username = ACCESS_KEY; key = SECRET_KEY -id = occi; type = OCCI; host = occiserver:4567; username = user; password = pass - -1.4.1 SECURITY - -Security is disabled by default, but it should be taken into account that it would -be possible that someone that has local network access can "sniff" the traffic and -get the messages with the IM with the authorisation data with the cloud providers. - -I can be activated both in the XMLRPC and REST APIs. Setting this variables: - -XMLRCP_SSL = True - -And then set the variables: XMLRCP_SSL_CA_CERTS to your CA certificates paths. - diff --git a/README b/README index 5bd7d9a..e087cb1 100644 --- a/README +++ b/README @@ -19,22 +19,6 @@ infrastructure. This file is composed of a set of "key - value" pairs, where the user specifies the authorization data for all the components and cloud deployments available. File auth.dat shows examples of authorization data. -The list of "key" values that must be specified for each component are: - -- id: An optional field used to identify the virtual deployment. It must be unique - in the authorization data. -- type: The type of the component. It can be any of the components of the - architecture, such as the "InfrastructureManager", "VMRC" or any of - the cloud deployments currently supported by the IM: OpenNebula, EC2, - OpenStack, OCCI or LibVirt. -- username: The name of the user for the authentication. In the EC2 and OpenStack - cases it refers to the Access Key ID value. password: The password for - the authentication. In the EC2 and OpenStack cases it refers to the - Secret Access Key value. -- host: The address to the server in format "address:port" to specify the cloud - deployment to access. In the EC2 and in the system components (IM and VMRC) - this field is not used. - To avoid typing the parameters in all the client calls. The user can define a config file "im_client.cfg" in the current directory or a file ".im_client.cfg" in their home directory. In the config file the user can specify the following parameters: diff --git a/README.md b/README.md index 2fba2c9..de180ab 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ usage: client.py [-u|--xmlrpc-url ] [-a|--auth_file ] operation o IM is based on python, so Python 2.4 or higher runtime and standard library must be installed in the system. -It is also required to install the Python Lex & Yacc library (http://www.dabeaz.com/ply/). -It is available in all of the main distributions as 'python-ply' package. +It is also required to install the RADL parser (https://github.com/grycap/radl), available in pip +as the 'RADL' package. 1.2 OPTIONAL PACKAGES -------------- @@ -70,57 +70,89 @@ xmlrpc_ssl_ca_certs=/tmp/pki/ca-chain.pem ### 1.4.1 AUTH FILE -The authorization data is used to validate access to the components in the -infrastructure. This file is composed of a set of "key - value" pairs, -where the user specifies the authorization data for all the components and cloud -deployments available. File auth.dat shows examples of authorization data. - -The list of "key" values that must be specified for each component are: - -* id: An optional field used to identify the virtual deployment. It must be unique - in the authorization data. -* type: The type of the component. It can be any of the components of the - architecture, such as the "InfrastructureManager", "VMRC" or any of - the cloud deployments currently supported by the IM: OpenNebula, EC2, - OpenStack, OCCI, LibCloud, GCE or LibVirt. -* username: The name of the user for the authentication. In the EC2 and OpenStack - cases it refers to the Access Key ID value. password: The password for - the authentication. In the EC2 and OpenStack cases it refers to the - Secret Access Key value. In the GCE case it can refer to the CLIENT_ID - and CLIENT_SECRET (using Installed Application authentication) or - the SERVICE_ACCOUNT_EMAIL and RSA_PRIVATE_KEY (using the Service - Account authentication) -* host: The address to the server in format "address:port" to specify the cloud - deployment to access. In the EC2 or GCE and in the system components (IM and VMRC) - this field is not used. +The authorization file stores in plain text the credentials to access the +cloud providers, the IM service and the VMRC service. Each line of the file +is composed by pairs of key and value separated by semicolon, and refers to a +single credential. The key and value should be separated by " = ", that is +**an equals sign preceded and followed by one white space at least**, like +this:: + + id = id_value ; type = value_of_type ; username = value_of_username ; password = value_of_password + +Values can contain "=", and "\\n" is replaced by carriage return. The available +keys are: + +* ``type`` indicates the service that refers the credential. The services + supported are ``InfrastructureManager``, ``VMRC``, ``OpenNebula``, ``EC2``,, ``FogBow``, + ``OpenStack``, ``OCCI``, ``LibCloud``, ``Docker``, ``GCE``, ``Azure``, ``Kubernetes`` and ``LibVirt``. + +* ``username`` indicates the user name associated to the credential. In EC2 + it refers to the *Access Key ID*. In Azure it refers to the user + Subscription ID. In GCE it refers to *Service Account’s Email Address*. + +* ``password`` indicates the password associated to the credential. In EC2 + it refers to the *Secret Access Key*. In GCE it refers to *Service + Private Key*. See how to get it and how to extract the private key file from + `here info `_). + +* ``tenant`` indicates the tenant associated to the credential. + This field is only used in the OpenStack plugin. + +* ``host`` indicates the address of the access point to the cloud provider. + This field is not used in IM and EC2 credentials. + +* ``proxy`` indicates the content of the proxy file associated to the credential. + To refer to a file you must use the function "file(/tmp/proxyfile.pem)" as shown in the example. + This field is only used in the OCCI plugin. + +* ``project`` indicates the project name associated to the credential. + This field is only used in the GCE plugin. + +* ``public_key`` indicates the content of the public key file associated to the credential. + To refer to a file you must use the function "file(cert.pem)" as shown in the example. + This field is only used in the Azure plugin. See how to get it + `here `_ + +* ``private_key`` indicates the content of the private key file associated to the credential. + To refer to a file you must use the function "file(key.pem)" as shown in the example. + This field is only used in the Azure plugin. See how to get it + `here `_ + +* ``id`` associates an identifier to the credential. The identifier should be + used as the label in the *deploy* section in the RADL. + +#### OpenStack addicional fields + +OpenStack has a set of addicional fields to access a cloud site: + +* ``auth_version`` the auth version used to connect with the Keystone server. + The possible values are: ``2.0_password`` or ``3.X_password``. The default value is ``2.0_password``. + +* ``base_url`` base URL to the OpenStack API endpoint. By default, the connector obtains API endpoint URL from the + server catalog, but if this argument is provided, this step is skipped and the provided value is used directly. + The value is: http://cloud_server.com:8774/v2/. + +* ``service_region`` the region of the cloud site (case sensitive). It is used to obtain the API + endpoint URL. The default value is: ``RegionOne``. + +* ``service_name`` the service name used to obtain the API endpoint URL. The default value is: ``Compute``. + +* ``auth_token`` token which is used for authentication. If this argument is provided, normal authentication + flow is skipped and the OpenStack API endpoint is directly hit with the provided token. Normal authentication + flow involves hitting the auth service (Keystone) with the provided username and password and requesting an + authentication token. An example of the auth file: ``` -id = one; type = OpenNebula; host = server:2633; username = user; password = pass -type = InfrastructureManager; username = user; password = pass -type = VMRC; host = http://server:8080/vmrc; username = user; password = pass -id = ec2; type = EC2; username = ACESS_KEY; password = SECRET_KEY -id = gce; type = GCE; username = CLIENT_ID; password = CLIENT_SECRET; project = project-name -id = gce2; type = GCE; username = SERVICE_ACC_EMAIL; password = file(path_to_pem_file); project = project-name -id = docker; type = Docker; host = server:2375 -id = occi; type = OCCI; host = server:8443; proxy = file(/tmp/proxy.pem) -id = libcloud; type = LibCloud; driver = EC2; username = ACESS_KEY; password = SECRET_KEY - -``` - -### 1.4.2 SECURITY - -Security is disabled by default, but it should be taken into account that it would -be possible that someone that has local network access can "sniff" the traffic and -get the messages with the IM with the authorisation data with the cloud providers. - -I can be activated both in the XMLRPC and REST APIs. Setting this variables: - +id = one; type = OpenNebula; host = osenserver:2633; username = user; password = pass +id = ost; type = OpenStack; host = https://ostserver:5000; username = user; password = pass; tenant = tenant +id = im; type = InfrastructureManager; username = user; password = pass +id = vmrc; type = VMRC; host = http://server:8080/vmrc; username = user; password = pass +id = ec2; type = EC2; username = ACCESS_KEY; password = SECRET_KEY +id = gce; type = GCE; username = username.apps.googleusercontent.com; password = pass; project = projectname +id = docker; type = Docker; host = http://host:2375 +id = occi; type = OCCI; proxy = file(/tmp/proxy.pem); host = https://fc-one.i3m.upv.es:11443 +id = azure; type = Azure; username = subscription-id; public_key = file(cert.pem); private_key = file(key.pem) +id = kub; type = Kubernetes; host = http://server:8080; username = user; password = pass ``` -XMLRCP_SSL = True -``` - -And then set the variables: XMLRCP_SSL_CA_CERTS to your CA certificates paths. - - diff --git a/changelog b/changelog index 57b095c..48e6c02 100644 --- a/changelog +++ b/changelog @@ -61,3 +61,6 @@ IM-client 1.4.2 * Improve returned messages * Bugfix in sshvm function in VMs using a port different to 22 +IM-client 1.4.3 + * Add GetVersion function + * Extract RADL as an external package diff --git a/doc/source/client.rst b/doc/source/client.rst index af22e4c..f51ecab 100644 --- a/doc/source/client.rst +++ b/doc/source/client.rst @@ -127,7 +127,7 @@ Values can contain "=", and "\\n" is replaced by carriage return. The available keys are: * ``type`` indicates the service that refers the credential. The services - supported are ``InfrastructureManager``, ``VMRC``, ``OpenNebula``, ``EC2``, + supported are ``InfrastructureManager``, ``VMRC``, ``OpenNebula``, ``EC2``,, ``FogBow``, ``OpenStack``, ``OCCI``, ``LibCloud``, ``Docker``, ``GCE``, ``Azure``, ``Kubernetes`` and ``LibVirt``. * ``username`` indicates the user name associated to the credential. In EC2 @@ -164,15 +164,36 @@ keys are: * ``id`` associates an identifier to the credential. The identifier should be used as the label in the *deploy* section in the RADL. + +OpenStack addicional fields +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +OpenStack has a set of addicional fields to access a cloud site: + +* ``auth_version`` the auth version used to connect with the Keystone server. + The possible values are: ``2.0_password`` or ``3.X_password``. The default value is ``2.0_password``. + +* ``base_url`` base URL to the OpenStack API endpoint. By default, the connector obtains API endpoint URL from the + server catalog, but if this argument is provided, this step is skipped and the provided value is used directly. + The value is: http://cloud_server.com:8774/v2/. + +* ``service_region`` the region of the cloud site (case sensitive). It is used to obtain the API + endpoint URL. The default value is: ``RegionOne``. + +* ``service_name`` the service name used to obtain the API endpoint URL. The default value is: ``Compute``. + +* ``auth_token`` token which is used for authentication. If this argument is provided, normal authentication + flow is skipped and the OpenStack API endpoint is directly hit with the provided token. Normal authentication + flow involves hitting the auth service (Keystone) with the provided username and password and requesting an + authentication token. An example of the auth file:: id = one; type = OpenNebula; host = osenserver:2633; username = user; password = pass - id = ost; type = OpenStack; host = ostserver:5000; username = user; password = pass; tenant = tenant - type = InfrastructureManager; username = user; password = pass - type = VMRC; host = http://server:8080/vmrc; username = user; password = pass + id = ost; type = OpenStack; host = https://ostserver:5000; username = user; password = pass; tenant = tenant + id = im; type = InfrastructureManager; username = user; password = pass + id = vmrc; type = VMRC; host = http://server:8080/vmrc; username = user; password = pass id = ec2; type = EC2; username = ACCESS_KEY; password = SECRET_KEY - id = oshost; type = OpenStack; host = oshost:8773; username = ACCESS_KEY; key = SECRET_KEY id = gce; type = GCE; username = username.apps.googleusercontent.com; password = pass; project = projectname id = docker; type = Docker; host = http://host:2375 id = occi; type = OCCI; proxy = file(/tmp/proxy.pem); host = https://fc-one.i3m.upv.es:11443 diff --git a/im_client.py b/im_client.py index 4efccdc..639c13f 100755 --- a/im_client.py +++ b/im_client.py @@ -23,7 +23,7 @@ from optparse import OptionParser, Option, IndentedHelpFormatter import ConfigParser -__version__ = "1.4.2" +__version__ = "1.4.3" class PosOptionParser(OptionParser): def format_help(self, formatter=None): @@ -232,25 +232,27 @@ def get_input_params(radl): parser.add_operation_help('startvm',' ') parser.add_operation_help('stopvm',' ') parser.add_operation_help('sshvm',' ') + parser.add_operation_help('getversion','') (options, args) = parser.parse_args() - if options.auth_file is None: - parser.error("Auth file not specified") - - auth_data = read_auth_data(options.auth_file) - - if auth_data is None: - parser.error("Auth file with incorrect format.") - if len(args) < 1: parser.error("operation not specified. Use --help to show all the available operations.") operation = args[0].lower() args = args[1:] - if (operation not in ["removeresource", "addresource", "create", "destroy", "getinfo", "list", "stop", "start", "alter", "getcontmsg", "getvminfo", "reconfigure","getradl","getvmcontmsg","stopvm","startvm", "sshvm","getstate"]): + if (operation not in ["removeresource", "addresource", "create", "destroy", "getinfo", "list", "stop", "start", "alter", "getcontmsg", "getvminfo", "reconfigure","getradl","getvmcontmsg","stopvm","startvm", "sshvm","getstate","getversion"]): parser.error("operation not recognised. Use --help to show all the available operations") + if (operation not in ["getversion"]): + if options.auth_file is None: + parser.error("Auth file not specified") + + auth_data = read_auth_data(options.auth_file) + + if auth_data is None: + parser.error("Auth file with incorrect format.") + if XMLRCP_SSL: print "Secure connection with: " + options.xmlrpc from springpython.remoting.xmlrpc import SSLClient @@ -583,3 +585,12 @@ def get_input_params(radl): else: print "Error accessing VM: %s" % info sys.exit(1) + + elif operation == "getversion": + (success, version) = server.GetVersion() + + if success: + print "IM service version: %s" % version + else: + print "ERROR getting IM service version: " + version + sys.exit(1) \ No newline at end of file diff --git a/radl/__init__.py b/radl/__init__.py deleted file mode 100644 index dc055b7..0000000 --- a/radl/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# IM - Infrastructure Manager -# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -__all__ = ['radl_lex','radl_parse','radl'] diff --git a/radl/radl.py b/radl/radl.py deleted file mode 100644 index be1810c..0000000 --- a/radl/radl.py +++ /dev/null @@ -1,1338 +0,0 @@ -# IM - Infrastructure Manager -# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from netaddr import IPNetwork, IPAddress -import copy -from distutils.version import LooseVersion - -def UnitToValue(unit): - """Return the value of an unit.""" - - if not unit: - return 1 - unit = unit[0].upper() - if unit == "K": - return 1024 - if unit == "M": - return 1024 * 1024 - if unit == "G": - return 1024 * 1024 * 1024 - return 1 - -def is_version(version, _): - return all([num.isdigit() for num in version.getValue().split(".")]) - -def check_outports_format(outports, _): - """ - Check the format of the outports string. - Valid formats: - 8899/tcp-8899/tcp,22/tcp-22/tcp - 8899/tcp-8899,22/tcp-22 - 8899-8899,22-22 - 8899/tcp,22/udp - 8899,22 - """ - try: - network.parseOutPorts(outports.getValue()) - except: - return False - else: - return True - -class RADLParseException(Exception): - """Error parsing RADL document.""" - - def __init__(self, msg="", line=None): - if line: - msg = "Line %d: %s" % (line, msg) - self.line = line - Exception.__init__(self, msg) - -class Feature: - """ - Every property that can appear in the definitions of a ``network`` and ``system``. - - Args: - - prop: feature name. - - operator: ``<=``, ``=``, ``=>`` or ``contains``. - - value: value associated to the feature. - - unit: like ``K``, ``M`` and ``G``. - - line: line number in the RADL document. - """ - - def __init__(self, prop = None, operator = None, value = None, unit = '', line=None): - self.prop = prop - self.operator = operator - self.value = value - self.unit = unit - self.line = line - - def __str__(self): - if isinstance(self.value, list): - return "{0} {1} ['{2}']".format(self.prop, self.operator, "','".join(self.value)) - else: - return ("{0} {1} ({2})" if self.operator == "contains" else - "{0} {1} '{2}'" if isinstance(self.value, str) or isinstance(self.value, unicode) else - "{0} {1} {2}{3}").format(self.prop, self.operator, self.value, - self.unit if self.unit else "") - - def clone(self): - """Return a copy of this Feature.""" - - return copy.deepcopy(self) - - def getLogOperator(self): - """Return the operator of this Feature in python style.""" - if self.operator == "=": - return "==" - else: - return self.operator - - def getValue(self, unit=None): - """ - Return the value of the feature. - - If the unit is specified and the feature has a unit, the value is converted - - Args: - - unit(str,optional): A unit to convert the current feature value ('B','K','M','G') - """ - if unit or self.unit: - r = float(self.value * UnitToValue(self.unit)) / UnitToValue(unit) - return int(round(r)) if isinstance(self.value, int) else r - return self.value - - def _check(self, check, radl): - """ - Check type, operator and unit in a feature. - - Args: - - check(tuple): - - v[0]: expected type of the feature value. - - v[1]: can be a list of possible values or a function to test the value or None. - - v[2] (optional): can be a list of possible units; if None or not set the - unit valid is none. - - radl: second argument passed when calling v[1]. - """ - - # Check type - # If the value must be float, int is also valid - if check[0] == float: - if not isinstance(self.value, int) and not isinstance(self.value, float): - raise RADLParseException("Invalid type; expected %s" % check[0], - line=self.line) - else: - if not isinstance(self.value, check[0]): - raise RADLParseException("Invalid type; expected %s" % check[0], - line=self.line) - # Check operator - if isinstance(self.value, str) and self.prop.find('version') == -1: - if self.operator != "=": - raise RADLParseException("Invalid operator; expected '='", - line=self.line) - elif isinstance(self.value, int) or isinstance(self.value, float) or self.prop.find('version') >= 0: - if self.operator not in ["=", "<=", ">=", ">", "<"]: - raise RADLParseException("Invalid operator; expected '=', '<=', " + - "'>=', '>' or '<'", line=self.line) - elif isinstance(self.value, Features): - if self.operator != "contains": - raise RADLParseException( - "Invalid operator; expected 'contains'", line=self.line) - # Check value - if isinstance(check[1], list): - if self.value.upper() not in check[1]: - raise RADLParseException("Invalid value; expected one of %s" % check[1], - line=self.line) - elif callable(check[1]): - if not check[1](self, radl): - raise RADLParseException("Invalid value in property '%s'" % self.prop, line=self.line) - # Check unit - if len(check) < 3 or check[2] == None: - if self.unit: - raise RADLParseException("Invalid unit; expected none", line=self.line) - elif len(check) > 2 and check[2]: - if self.unit.upper() not in check[2]: - raise RADLParseException( - "Invalid unit; expected one of %s" % check[2], line=self.line) - return True - -class Features(object): - """ - Collects a group of features. - """ - - def __init__(self, features=None): - self.props = {} - if features: - for f in features: - self.addFeature(f) - - @property - def features(self): - """List of features.""" - - r = [] - for _, inter in self.props.items(): - if isinstance(inter, tuple): - if (inter[0] and inter[1] and inter[0].getValue() == inter[1].getValue() and - inter[0].operator == "=" and inter[1].operator == "="): - r.append(inter[0]) - else: - r.extend([f for f in inter if f]) - elif isinstance(inter, dict): - r.extend(inter.values()) - elif isinstance(inter, list): - r.extend(inter) - else: - r.append(inter) - return r - - def __str__(self): - return " and\n".join([str(f) for f in self.features]) - - def __eq__(self, other): - if other is None: - return self is None - return self.props == other.props - - def clone(self): - """Return a copy of this aspect.""" - - return copy.deepcopy(self) - - def addFeature(self, f, conflict="error", missing="other"): - """ - Add a feature. - - Args: - - - f(Feature): feature to add. - - conflict(str): if a property hasn't compatible values/constrains, do: - - ``"error"``: raise exception. - - ``"ignore"``: go on. - - ``"me"``: keep the old value. - - ``"other"``: set the passed value. - - missing(str): if a property has not been set yet, do: - - ``"error"``: raise exception. - - ``"ignore"``: do nothning. - - ``"me"``: do nothing. - - ``"other"``: set the passed value. - """ - - OPTIONS = ["error", "ignore", "me", "other"] - assert missing in OPTIONS, "Invalid value in `missing`." - assert conflict in OPTIONS, "Invalid value in `missing`." - - if f.prop not in self.props and missing == "error": - raise Exception("Property has not set.") - elif f.prop not in self.props and missing in ["ignore", "first"]: - return - - if isinstance(f.value, int) or isinstance(f.value, float): - if f.operator == "=": - inter1 = (f, f) - elif f.operator[0] == "<": - inter1 = (None, f) - elif f.operator[0] == ">": - inter1 = (f, None) - inter0 = self.props.get(f.prop, (None, None)) - try: - self.props[f.prop] = Features._applyInter(inter0, inter1, conflict) - except Exception, e: - raise RADLParseException("%s. Involved features: %s" % (e, [str(f0) for f0 in inter0]), - line=f.line) - elif isinstance(f, SoftFeatures): - self.props.setdefault(f.prop, []).append(f) - elif f.operator == "contains": - if f.prop in self.props and f.value.getValue("name") in self.props[f.prop]: - feature = self.props[f.prop][f.value.getValue("name")].clone() - for f0 in f.value.features: - feature.value.addFeature(f0, conflict, missing) - self.props[f.prop][f.value.getValue("name")] = feature - else: - self.props.setdefault(f.prop, {})[f.value.getValue("name")] = f - else: - value0 = self.props.get(f.prop, None) - if not value0 or (conflict == "other"): - self.props[f.prop] = f - elif value0.value != f.value and conflict == "error": - raise RADLParseException("Conflict adding `%s` because `%s` is already set." % (f, value0), line=f.line) - - def hasFeature(self, prop, check_softs=False): - """Return if there is a property with that name.""" - - return prop in self.props or (check_softs and - any([ fs.hasFeature(prop) for fs in self.props.get(SoftFeatures.SOFT, []) ])) - - def getValue(self, prop, default=None): - """Return the value of feature with that name or ``default``.""" - - f = self.props.get(prop, None) - if not f: - return default - if isinstance(f, Feature): - return f.getValue() - if isinstance(f, tuple): - #if f[0].getValue() == f[1].getValue(): - # return f[0].getValue() - # Miguel: para poder probar - if f[0]: - return f[0].getValue() - elif f[1]: - return f[1].getValue() - raise Exception("Getting value from a property with a constrain") - if isinstance(f, list): - return f - raise Exception("Getting value from a not simple property.") - - def getFeature(self, prop): - """Return the feature with that name.""" - - f = self.props.get(prop, None) - if not f: - return None - if isinstance(f, Feature): - return f - if isinstance(f, tuple): - if f[0]: - return f[0] - elif f[1]: - return f[1] - raise Exception("Getting value from a property with a constrain") - if isinstance(f, list): - return f - raise Exception("Getting value from a not simple property.") - - def setValue(self, prop, value, unit=None): - """Set the value of feature with that name.""" - - if isinstance(value, int) or isinstance(value, float): - if prop in self.props: - for i, j in [(0, 1), (1, 0)]: - if self.props[prop][i] == None: - self.props[prop] = (self.props[prop][j], self.props[prop][j]) - for v in self.props[prop]: - v.value, v.unit = value, unit - else: - f = Feature(prop, "=", value, unit=unit) - self.props[prop] = (f, f) - elif prop in self.props: - self.props[prop].value, self.props[prop].unit = value, unit - else: - self.props[prop] = Feature(prop, "=", value, unit=unit) - - def delValue(self, prop): - """Remove the feature with that name.""" - - try: - del self.props[prop] - except: - pass - - @staticmethod - def _applyInter(finter0, finter1, conflict="ignore"): - """ - Return the restriction of first interval by the second. - - Args: - - - inter0, inter1 (tuple of Feature): intervals - - Return(tuple of Feature): the resulting interval - - conflict(str): if a property hasn't compatible values/constrains, do: - - ``"error"``: raise exception. - - ``"ignore"``: return None. - - ``"me"``: return finter0. - - ``"other"``: return finter1. - """ - - OPTIONS = ["error", "ignore", "me", "other"] - assert conflict in OPTIONS, "Invalid value in `conflict`." - - # Compute the comparison of the interval extremes - # Remember, None <= number and None <= None are True, but number <= None is False. - inter0 = tuple([f.getValue() if f else None for f in finter0]) - inter1 = tuple([f.getValue() if f else None for f in finter1]) - le00 = inter0[0] <= inter1[0] # finter0[0] <= finter1[0] - le01 = inter1[1] == None or inter0[0] <= inter1[1] # finter0[0] <= finter1[1] - le11 = inter1[1] == None or (inter0[1] != None and inter0[1] <= inter1[1]) - # finter0[1] <= finter1[1] - ge00 = not le00 or inter0[0] == inter1[0] # finter0[0] >= finter1[0] - ge10 = inter0[1] == None or inter0[1] >= inter1[0] # finter0[1] >= finter1[0] - - #print "\n".join("%s: %s" % (s, v) for v, s in [ - # (le00, "finter0[0] <= finter1[0]"), - # (le01, "finter0[0] <= finter1[1]"), - # (le11, "finter0[1] <= finter1[1]"), - # (ge00, "finter0[0] >= finter1[0]"), - # (ge10, "finter0[1] >= finter1[0]") ]) - - # First interval is ( ), second interval is [ ] - if le00 and ge10 and le11: # ( [ ) ] chain first-second - return finter1[0], finter0[1] - elif le00 and ge10 and not le11: # ( [ ] ) second is inside first - return finter1 - elif ge00 and le01 and le11: # [ ( ) ] first is inside second - return finter0 - elif ge00 and le01 and not le11: # [ ( ] ) chain second-first - return finter0[0], finter1[1] - elif conflict == "me": - return finter0 - elif conflict == "other": - return finter1 - elif conflict == "error": - raise Exception("Disjoint intervals!") - return None - - def applyFeatures(self, new_features, conflict="error", missing="error"): - """ - Apply the constrain of the features passed to this instance. - - .. warning:: - Feature instances are only considered, that is, SoftFeatures will be - not considered. - - Args: - - - new_features(Features): features to apply - - conflict(str): if a property hasn't compatible values/constrains, do: - - ``"error"``: raise exception. - - ``"ignore"``: nothing. - - ``"me"``: preserve the original value. - - ``"other"``: set like the passed feature. - - missing(str): if a property is missing in some side, do: - - ``"error"``: raise exception. - - ``"ignore"``: nothing. - - ``"me"``: preserve the original value. - - ``"other"``: set like the passed feature. - """ - - OPTIONS = ["error", "ignore", "me", "other"] - assert missing in OPTIONS, "Invalid value in `missing`." - assert conflict in OPTIONS, "Invalid value in `missing`." - - self0 = self.clone() - if isinstance(new_features, Features): - new_features = new_features.features - for f in new_features: - self0.addFeature(f, conflict=conflict, missing=missing) - self.props = self0.props - return self - - def check_simple(self, checks, radl): - """Check types, operators and units in simple features.""" - - for f in self.features: - if not isinstance(f, Feature) or f.prop not in checks: continue - f._check(checks[f.prop], radl) - - def check_num(self, checks, radl): - """ - Check types, operators and units in features with numbers. - - Args: - - - checks(dict of dict of str:tuples): keys are property name prefixes, and the - values are dict with keys are property name suffixes and values are iterable - as in ``_check_feature``. - - radl: passed to ``_check_feature``. - """ - - prefixes = {} - for f in self.features: - if not isinstance(f, Feature): continue - (prefix, sep, tail) = f.prop.partition(".") - if not sep or prefix not in checks: continue - checks0 = checks[prefix] - (num, sep, suffix) = tail.partition(".") - try: - num = int(num) - except: - raise RADLParseException( - "Invalid property name; expected an index.", line=f.line) - if not sep or suffix not in checks0: continue - f._check(checks0[suffix], radl) - if prefix not in prefixes: prefixes[prefix] = set() - prefixes[prefix].add(num) - - # Check consecutive indices for num properties. - for prefix, nums in prefixes.items(): - if min(nums) != 0 or max(nums) != len(nums)-1: - raise RADLParseException( - "Invalid indices values in properties '%s'" % prefix) - - return prefixes - -class Aspect: - """A network, system, deploy, configure or contextualize element in a RADL.""" - - def getId(self): - """Return the id of the aspect.""" - - return id(self) - - def clone(self): - """Return a copy of this aspect.""" - - return copy.deepcopy(self) - -class contextualize_item: - """Store a line under ``contextualize`` RADL keyword.""" - def __init__(self, system_id, configure_id, num=0, ctxt_tool=None ,line=None): - self.system = system_id - """System id.""" - self.configure = configure_id - """Configure id.""" - self.num = num - """Num of steps (optional).""" - self.ctxt_tool = ctxt_tool - """Name of the Ctxt. tool (optional). Currently supported: 'Ansible' and 'cloud-init'. Default 'Ansible'.""" - self.line = line - - def __str__(self): - return "system %s configure %s %s %s" % (self.system, self.configure, - "step " + str(self.num) if self.num else "", - "with " + self.ctxt_tool if self.ctxt_tool else "") - - def getId(self): - """Return an unique key for this element.""" - - return (self.system, self.configure) - - def get_ctxt_tool(self): - """Return the name of the Ctxt. tool.""" - - return self.ctxt_tool if self.ctxt_tool else "Ansible" - - def check(self, radl): - """Check a line under a contextualize.""" - - if not radl.get_system_by_name(self.system): - raise RADLParseException("Invalid system id '%s'" % self.system, line=self.line) - if not radl.get_configure_by_name(self.configure): - raise RADLParseException("Invalid configure id '%s'" % self.configure, line=self.line) - - -class contextualize(Aspect, object): - """Store a ``contextualize`` RADL keyword.""" - def __init__(self, items=None, max_time=0, line=None): - self.max_time = max_time - """Maximum time.""" - self.items = None - """List of contextualize_item.""" - if isinstance(items, list): - self.items = dict([(c.getId(), c) for c in items]) - elif isinstance(items, dict): - self.items = items - elif items is not None: - raise ValueError("Unexpected type for 'items'.") - self.line = line - - def __str__(self): - if self.items is None: - return "" - elif not self.items: - return "contextualize ()" - else: - return "contextualize %s (\n%s\n)" % (self.max_time if self.max_time else "", - "\n".join([str(i) for i in self.items.values()])) - - def __len__(self): - if self.items is None: - return 0 - else: - return len(self.items) - - def update(self, cont): - """Update this instance with the contextualize passed.""" - - self.max_time = max(self.max_time, cont.max_time) - if self.items is None: - self.items = cont.items - else: - self.items.update(cont.items) - - def check(self, radl): - """Check a contextualize.""" - - if not isinstance(self.max_time, int) or self.max_time < 0: - raise RADLParseException("Invalid 'max time' in 'contextualize'", - line=self.line) - if self.items is not None: - for i in self.items.values(): - i.check(radl) - - def get_contextualize_items_by_step(self, default=None): - """Get a dictionary of the contextualize_items grouped by the step or the default value""" - if self.items: - res = {} - for elem in self.items.values(): - if elem.num in res: - res[elem.num].append(elem) - else: - res[elem.num] = [elem] - return res - else: - return default - - -class configure(Aspect): - """Store a RADL ``configure``.""" - - def __init__(self, name, recipe="", reference=False, line=None): - self.recipes = recipe - """Recipe content.""" - self.name = name - """Configure id.""" - self.reference = reference - """True if it is only a reference and it isn't a definition.""" - self.line = line - - def getId(self): - return self.name - - def __str__(self): - if self.reference or not self.recipes: - return "configure %s" % self.name - return "configure %s (\n@begin\n%s\n@end\n)" % (self.name, self.recipes) - - def check(self, _): - """Check this configure.""" - - try: - import yaml - except: - return True - try: - yaml.load(self.recipes) - except Exception, e: - raise RADLParseException("Invalid YAML code: %s." % e, line=self.line) - return True - -class deploy(Aspect): - """Store a RADL ``deploy``.""" - - def __init__(self, deploy_id, vm_number, cloud_id=None, line=None): - self.id = deploy_id - """System id.""" - self.vm_number = vm_number - """Number of virtual machines to deploy.""" - self.cloud_id = cloud_id - """Cloud provider id.""" - self.line = line - - def __str__(self): - res = "deploy " + self.id + (" %s" % self.vm_number) - if self.cloud_id: - res += " " + self.cloud_id - return res - - def check(self, radl): - """Check this deploy.""" - - if not radl.get_system_by_name(self.id): - raise RADLParseException("Invalid system id in the deploy.", line=self.line) - - if self.vm_number < 0: - raise RADLParseException("Invalid number of virtual machines to deploy.", - line=self.line) - -class network(Features, Aspect): - """Store a RADL ``network``.""" - - private_net_masks = ["10.0.0.0/8","172.16.0.0/12","192.168.0.0/16","169.254.0.0/16","100.64.0.0/10","192.0.0.0/24","198.18.0.0/15"] - - def __init__(self, name, features=None, reference=False, line=None): - self.id = name - """Network id.""" - self.reference = reference - """True if it is only a reference and it isn't a definition.""" - Features.__init__(self, features) - self.line = line - - @staticmethod - def isPrivateIP(ip): - """ - Check if an IP address is private - """ - for mask in network.private_net_masks: - if IPAddress(ip) in IPNetwork(mask): - return True - return False - - def getId(self): - return self.id - - def __str__(self): - return "network %s %s" % (self.id, "" if self.reference else "( %s )" % Features.__str__(self)) - - def check(self, radl): - """Check the features in this network.""" - - SIMPLE_FEATURES = { - "outbound": (str, ["YES", "NO"]), - "outports": (str, check_outports_format) - } - self.check_simple(SIMPLE_FEATURES, radl) - - def isPublic(self): - """Return true if outbound = yes.""" - return self.getValue("outbound") == "yes" - - @staticmethod - def createNetwork(name, public=False): - """Return a network with id being ``name`` and with outbound=yes if ``public``.""" - - return network(name, [Feature("outbound", "=", "yes" if public else "no")]) - - @staticmethod - def parseOutPorts(outports): - """ - Parse the outports string - Valid formats: - 8899/tcp-8899/tcp,22/tcp-22/tcp - 8899/tcp-8899,22/tcp-22 - 8899-8899,22-22 - 8899/tcp,22/udp - 8899,22 - Returns a list of tuple with the format: (remote_port,remote_protocol,local_port,local_protocol) - """ - res = [] - ports = outports.split(',') - for port in ports: - parts = port.split('-') - remote_port = parts[0] - if len(parts) > 1: - local_port = parts[1] - else: - local_port = remote_port - - local_port_parts = local_port.split("/") - if len(local_port_parts) > 1: - local_protocol = local_port_parts[1] - local_port = local_port_parts[0] - else: - local_protocol = "tcp" - - remote_port_parts = remote_port.split("/") - if len(remote_port_parts) > 1: - remote_protocol = remote_port_parts[1] - remote_port = remote_port_parts[0] - else: - remote_protocol = "tcp" - res.append((int(remote_port),remote_protocol,int(local_port),local_protocol)) - return res - - def getOutPorts(self): - """ - Get the outports of this network. - outports format: 22/tcp-22/tcp,8899/tcp,8800 - Returns a list of tuples with the format: (remote_port,remote_protocol,local_port,local_protocol) - """ - outports = self.getValue("outports") - if outports: - return self.parseOutPorts(outports) - else: - return None - -class FeaturesApp(Features): - """Store an RADL application.""" - - def __init__(self, features): - Features.__init__(self, features) - - @staticmethod - def from_str(app_name, app_version = None): - if app_name != None: - res = FeaturesApp([]) - res.addFeature(Feature(prop = "name", operator = "=", value = app_name)) - if app_version: - res.addFeature(Feature(prop = "version", operator = "=", value = app_version)) - return res - else: - return None - - def isNewerThan(self, other): - """ Compare if the version of this app is newer that the other """ - if self.getValue("name") == other.getValue("name"): - if other.getValue("version"): - if not other.getValue("version"): - return False - else: - return LooseVersion(self.getValue("version")) > LooseVersion(other.getValue("version")) - else: - return True - else: - return False - - def check(self, radl): - """Check the features in this application.""" - SIMPLE_FEATURES = { - "name": (str, lambda x,_: bool(x.value)), - "path": (str, lambda x,_: bool(x.value)), - "version": (str, is_version), - "preinstalled": (str, ["YES", "NO"]) - } - self.check_simple(SIMPLE_FEATURES, radl) - -class Credentials: - pass - -class UserPassCredential(Credentials): - def __init__(self, user, passwd): - self.username = user - self.password = passwd - -class UserKeyCredential(Credentials): - def __init__(self, user, public, private=None): - self.username = user - self.public_key = public - self.private_key = private - -class system(Features, Aspect): - """Store a RADL ``system``.""" - - def __init__(self, name, features=None, reference=False, line=None): - self.name = name - """System id.""" - self.reference = reference - """True if it is only a reference and it isn't a definition.""" - Features.__init__(self, features) - self.line = line - - def getId(self): - return self.name - - def __str__(self): - return "system %s %s" % (self.name, "" if self.reference else "(\n%s\n)\n" % Features.__str__(self)) - - def hasIP(self, ip): - """Return True if some system has this IP.""" - - for f in self.features: - if (f.prop.startswith("net_interface.") and - f.prop.endswith(".ip") and f.value == ip): - return True - return False - - def getIfaceIP(self, iface_num): - """Return IP in the interface with that number.""" - - ip = self.getValue("net_interface.%d.ip" % iface_num) - if ip: - return ip - return None - - def getNumNetworkIfaces(self): - """Return the number of network interfaces defined.""" - - i = 0 - while self.hasFeature("net_interface.%d.connection" % i): - i += 1 - return i - - def getNumNetworkWithConnection(self, connection): - """Return the number of network interfaces with id ``connection``.""" - - i = 0 - while True: - value = self.getValue("net_interface.%d.connection" % i, None) - if not value: - return None - if value == connection: - return i - i += 1 - - def getRequestedNameIface(self, iface_num=0, num = None, default_hostname = None, default_domain = None): - """Return the dns name associated to the net interface.""" - - full_name = self.getValue("net_interface.%d.dns_name" % iface_num) - - if full_name: - replaced_full_name = system.replaceTemplateName(full_name, num) - (hostname, domain) = replaced_full_name - if not domain: - domain = default_domain - return (hostname, domain) - else: - if default_hostname: - (hostname, _) = system.replaceTemplateName(default_hostname, num) - return (hostname, default_domain) - else: - return None - - @staticmethod - def replaceTemplateName(full_name, num = None): - if full_name: - if num is not None: - full_name = full_name.replace("#N#", str(num)) - dot_pos = full_name.find('.') - if dot_pos != -1: - domain = full_name[dot_pos+1:] - name = full_name[:dot_pos] - return (name, domain) - else: - return (full_name, None) - else: - return full_name - - def getNetworkIDs(self): - """Return a list of network id of this system.""" - - res = [] - i = 0 - while True: - netid = self.getValue("net_interface.%d.connection" % i) - if not netid: - return res - res.append(netid) - i += 1 - - def getCredentialValues(self, new = False): - """Return the values in disk.0.os.credentials.*.""" - - credentials_base = "disk.0.os.credentials." - if new: - credentials_base = "disk.0.os.credentials.new." - return tuple([ self.getValue(credentials_base + p) for p in [ - "username", "password", "public_key", - "private_key"] - ]) - - def updateNewCredentialValues(self): - """ - Set the new credential values to the credentials to use, and delete the new ones - """ - - credentials_base = "disk.0.os.credentials." - new_credentials_base = "disk.0.os.credentials.new." - - for elem in ['password','public_key','private_key']: - if self.getValue(new_credentials_base + elem): - self.setValue(credentials_base + elem, self.getValue(new_credentials_base + elem)) - self.delValue(new_credentials_base + elem) - - def setCredentialValues(self, username=None, password=None, public_key=None, private_key=None, new = False): - """Set the values in disk.0.os.credentials.*.""" - - credentials_base = "disk.0.os.credentials." - if new: - credentials_base = "disk.0.os.credentials.new." - - if username: - self.setValue(credentials_base + "username", username) - if password: - self.setValue(credentials_base + "password", password) - if public_key: - self.setValue(credentials_base + "public_key", public_key) - if private_key: - self.setValue(credentials_base + "private_key", private_key) - - - def getCredentials(self): - """Return UserKeyCredential or UserPassCredential.""" - - (username, password, public_key, private_key) = self.getCredentialValues() - - if public_key or private_key: - return UserKeyCredential(username, public_key, private_key) - - if username or password: - return UserPassCredential(username, password) - - return None - - def setCredentials(self, creds): - """Set values in UserKeyCredential or UserPassCredential.""" - - if isinstance(creds, UserKeyCredential): - self.setUserKeyCredentials(creds.username, creds.public_key, creds.private_key) - elif isinstance(creds, UserPassCredential): - self.setUserPasswdCredentials(creds.username, creds.password) - - def setUserPasswdCredentials(self, username, password): - """Set username and password in ``disk.0.os.credentials``.""" - - self.setCredentialValues(username=username,password=password) - - def setUserKeyCredentials(self, username, public_key=None, private_key=None): - """Set these properties in ``disk.0.os.credentials``.""" - - self.setCredentialValues(username=username,public_key=public_key,private_key=private_key) - - def getApplications(self): - """Return a list of Application with the specified apps in this system.""" - - res = [] - for f in self.features: - if isinstance(f, Feature) and f.prop == "disk.0.applications": - res.append(FeaturesApp(f.value.features)) - - return res - - def addApplication(self, name, version=None, path=None, disk_num=0, soft=-1): - """Add a new application in some disk.""" - - fapp = Features() - fapp.features.append(Feature("name", "=", name)) - if version: - fapp.features.append(Feature("version", "=", version)) - if path: - fapp.features.append(Feature("path", "=", path)) - self.features.append( - Feature("disk.%d.applications" % disk_num, "contains", fapp, soft > 0)) - - def check(self, radl): - """Check the features in this system.""" - - def positive(f, _): - return f.value >= 0 - - mem_units = ["", "B", "K", "M", "G", "KB", "MB", "GB"] - SIMPLE_FEATURES = { - "spot": (str, ["YES", "NO"]), - "image_type": (str, ["VMDK", "QCOW", "QCOW2", "RAW"]), - "virtual_system_type": (str, system._check_virtual_system_type), - "price": ((int,float), positive, None), - "cpu.count": (int, positive, None), - "cpu.arch": (str, ['I386', 'X86_64']), - "cpu.performance": ((int,float), positive, ["ECU", "GCEU", "HRZ"]), - "memory.size": (int, positive, mem_units), - SoftFeatures.SOFT: (SoftFeatures, lambda x, r: x.check(r)) - } - self.check_simple(SIMPLE_FEATURES, radl) - - net_connections = set() - def check_net_interface_connection(f, radl0): - if radl0.get_network_by_id(f.value) == None: - return False - net_connections.add(f.prop) - return True - - def check_app(f, x): - FeaturesApp(f.value.features).check(x) - return True - - NUM_FEATURES = { - "net_interface": { - "connection": (str, check_net_interface_connection), - "dns_name": (str, None) }, - "disk": { - "image.url": ((str,list), system._check_disk_image_url), - "image.name": (str, None), - "type": (str, ["SWAP", "ISO", "FILESYSTEM"]), - "device": (str, None), - "size": (float, positive, mem_units), - "free_size": (float, positive, mem_units), - "os.name": (str, ["LINUX", "WINDOWS", "MAC OS X"]), - "os.flavour": (str, None), - "os.version": (str, is_version), - "os.credentials.username": (str, None), - "os.credentials.password": (str, None), - "os.credentials.private_key": (str, None), - "os.credentials.public_key": (str, None), - "applications": (Features, check_app) - } - } - prefixes = self.check_num(NUM_FEATURES, radl) - - # Check all interfaces - if len(net_connections) != len(prefixes.get("net_interface", set())): - raise RADLParseException( "Some net_interface does not have a connection") - - return True - - @staticmethod - def _check_disk_image_url(f, radl): - return True - - @staticmethod - def _check_virtual_system_type(f, radl): - return True - - def concrete(self, other=None): - """ - Return copy and score after being applied other system and soft features. - - Args: - - - other(system, optional): system to apply just before soft features. - - Return(tuple): tuple of the resulting system and its score. - """ - - new_system = self.clone() - if other: - new_system.applyFeatures(other, missing="other") - soft_features = self.getValue(SoftFeatures.SOFT, []) - score = 0 - for f in sorted(soft_features, key=lambda f: f.soft, reverse=True): - try: - new_system.applyFeatures(f, missing="other") - score += f.soft - except: - pass - new_system.delValue(SoftFeatures.SOFT) - return new_system, score - - -class SoftFeatures(system, Feature): - """ - Assign a weight to a group of features. - - Args: - - soft: weight of matching the containing features. - """ - - SOFT = "__soft__" - """Fake property name.""" - - def __init__(self, soft, features, line=None): - self.soft = soft - system.__init__(self, None, features, line=line) - Feature.__init__(self, SoftFeatures.SOFT, "contains", self) - - def __str__(self): - return "soft %s ( %s )" % (self.soft, Features.__str__(self)) - - -class RADL: - """Parsed RADL document.""" - - def __init__(self): - self.networks = [] - """List of network.""" - self.ansible_hosts = [] - """List of ansible_hosts.""" - self.systems = [] - """List of system.""" - self.deploys = [] - """List of deploy.""" - self.configures = [] - """List of configure.""" - self.contextualize = contextualize() - """List of contextualize.""" - - def __str__(self): - return "\n".join([ str(f) for fs in [self.networks, self.systems, self.configures, - [self.contextualize], self.deploys] for f in fs ]) - - def add(self, aspect, ifpresent="error"): - """ - Add a network, system, deploy, configure or contextualize. - - Args: - - aspect(network, system, deploy, configure or contextualize): thing to add. - - ifpresent(str): if it has been defined, do: - - - ``"ignore"``: not add the aspect. - - ``"replace"``: replace by the old defined. - - ``"error"``: raise an error. - - Return(bool): True if aspect was added. - """ - - # If aspect is a contextualization, it is trated separately - if isinstance(aspect, contextualize): - self.contextualize.update(aspect) - return True - - classification = [(network, self.networks), (system, self.systems), (ansible, self.ansible_hosts), - (deploy, self.deploys), (configure, self.configures)] - aspect_list = [l for t, l in classification if isinstance(aspect, t)] - assert len(aspect_list) == 1, "Unexpected aspect for RADL." - aspect_list = aspect_list[0] - - old_aspect = [a for a in aspect_list if a.getId() == aspect.getId()] - if old_aspect: - # If some aspect with the same id is found - if ifpresent == "error": - raise Exception("Aspect with the same id was found.") - elif ifpresent == "replace": - aspect_list.remove(old_aspect[0]) - aspect_list.append(aspect) - return True - elif ifpresent == "ignore": - return False - else: - raise ValueError - else: - # Otherwise add aspect - aspect_list.append(aspect) - return True - - def get(self, aspect): - """Get a network, system or configure or contextualize with the same id as aspect passed.""" - - classification = [(network, self.networks), (system, self.systems), - (configure, self.configures)] - aspect_list = [l for t, l in classification if isinstance(aspect, t)] - assert len(aspect_list) == 1, "Unexpected aspect for RADL." - aspect_list = aspect_list[0] - - old_aspect = [a for a in aspect_list if a.getId() == aspect.getId()] - return old_aspect[0] if old_aspect else None - - def clone(self): - return copy.deepcopy(self) - - def __getIP(self, public): - """Return the first net_interface.%d.ip for a system in a public/private network.""" - - maybeNot = (lambda x: x) if public else (lambda x: not x) - nets_id = [net.id for net in self.networks if maybeNot(net.isPublic())] - for s in self.systems: - i = 0 - while True: - value = s.getValue("net_interface.%d.connection" % i) - if not value: - break - if value in nets_id: - return s.getValue("net_interface.%d.ip" % i) - i += 1 - return None - - def getPublicIP(self): - """Return the first net_interface.%d.ip for a system in a public network.""" - - return self.__getIP(True) - - def getPrivateIP(self): - """Return the first net_interface.%d.ip for a system in a private network.""" - - return self.__getIP(False) - - - def hasPublicNet(self, system_name): - """ Return true if some system has a public network.""" - - nets_id = [net.id for net in self.networks if net.isPublic()] - system = self.get_system_by_name(system_name) - if system: - i = 0 - while True: - f = system.getFeature("net_interface.%d.connection" % i) - if not f: - break - if f.value in nets_id: - return True - i += 1 - - return False - - def check(self): - """Check if it is a valid RADL document.""" - - for i in [ f for fs in [self.networks, self.systems, self.deploys, - self.configures, [self.contextualize]] for f in fs ]: - i.check(self) - return True - - def get_system_by_name(self, name): - """Return a system with that name or None.""" - - for elem in self.systems: - if elem.name == name: - return elem - return None - - def get_deploy_by_id(self, dep_id): - """Return a deploy with that system id or None.""" - - for elem in self.deploys: - if elem.id == dep_id: - return elem - return None - - def get_configure_by_name(self, name): - """Return a configure with that id or None.""" - - for elem in self.configures: - if elem.name == name: - return elem - return None - - def get_network_by_id(self, net_id): - """Return a network with that id or None.""" - - for elem in self.networks: - if elem.id == net_id: - return elem - return None - - def get_ansible_by_id(self, ansible_id): - """Return a ansible with that id or None.""" - - for elem in self.ansible_hosts: - if elem.id == ansible_id: - return elem - return None - -class ansible(Features, Aspect): - """Store a RADL ``ansible``.""" - - def __init__(self, name, features, line=None): - self.id = name - """Ansible host id.""" - Features.__init__(self, features) - self.line = line - self.reference = False - - def __str__(self): - return "ansible %s (%s)" % (self.id, Features.__str__(self)) - - def check(self, radl): - """Check the features in this network.""" - - SIMPLE_FEATURES = { - "host": (str, None), - "credentials.username": (str, None), - "credentials.password": (str, None), - "credentials.private_key": (str, None) - } - self.check_simple(SIMPLE_FEATURES, radl) - - if not self.getHost(): - raise RADLParseException("Ansible host must have a host", line=self.line) - (username, password, private_key) = self.getCredentialValues() - if not username: - raise RADLParseException("Ansible host must have a credentials.username", line=self.line) - if not password and not private_key: - raise RADLParseException("Ansible host must have a credentials.password or credentials.private_key", line=self.line) - - def getHost(self): - return self.getValue("host") - - def getCredentials(self): - """Return UserKeyCredential or UserPassCredential.""" - - (username, password, private_key) = self.getCredentialValues() - - if private_key: - return UserKeyCredential(username, None, private_key) - - if username or password: - return UserPassCredential(username, password) - - return None - - def getCredentialValues(self, new = False): - """Return the values in credentials.*.""" - - credentials_base = "credentials." - return tuple([ self.getValue(credentials_base + p) for p in [ - "username", "password", "private_key"] - ]) \ No newline at end of file diff --git a/radl/radl_parse.py b/radl/radl_parse.py deleted file mode 100644 index 4702b35..0000000 --- a/radl/radl_parse.py +++ /dev/null @@ -1,383 +0,0 @@ -# IM - Infrastructure Manager -# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import ply.lex as lex -import ply.yacc as yacc - -import os -from radl import Feature, RADL, system, network, ansible, configure, contextualize, contextualize_item, \ - deploy, SoftFeatures, Features, RADLParseException - -class RADLParser: - - def __init__(self, autodefinevars = True, **kwargs): - self.lexer = lex.lex(module=self, debug=0, optimize=0, **kwargs) - self.yacc = yacc.yacc(module=self, debug=0, optimize=0) - - states = ( - ('recipe', 'exclusive'), - ('body', 'inclusive'), - ) - - # Lista de nombres de Token. Esto es obligatorio. - tokens = ( - 'LPAREN', - 'RPAREN', - 'LBRACK', - 'RBRACK', - 'COMMA', - 'NUMBER', - 'AND', - 'EQ', - 'LT', - 'GT', - 'GE', - 'LE', - 'SOFT', - 'STRING', - 'VAR', - 'CONTAINS', - 'DEPLOY', - 'CONFIGURE', - 'SYSTEM', - 'NETWORK', - 'ANSIBLE', - 'RECIPE_LINE', - 'RECIPE_BEGIN', - 'RECIPE_END', - 'CONTEXTUALIZE', - 'STEP', - 'WITH' - ) - - # A string containing ignored characters (spaces and tabs) - t_ignore = ' \t' - t_recipe_ignore = '' - t_body_ignore = ' \t' - - # Ignore comments. - def t_comment(self,t): - r'\#.*' - pass - - def t_body_LE(self,t): - r'<=' - return t - - def t_body_GE(self,t): - r'>=' - return t - - def t_body_EQ(self,t): - r'=' - return t - - def t_body_GT(self,t): - r'>' - return t - - def t_body_LT(self,t): - r'<' - return t - - def t_LPAREN(self,t): - r'\(' - t.lexer.push_state('body') - return t - - def t_RPAREN(self,t): - r'\)' - t.lexer.pop_state() - return t - - def t_body_LBRACK(self,t): - r'\[' - return t - - def t_body_RBRACK(self,t): - r'\]' - return t - - def t_body_COMMA(self,t): - r'\,' - return t - - def t_newline(self,t): - r'\n' - t.lexer.lineno += len(t.value) - - def t_body_newline(self,t): - r'\n' - t.lexer.lineno += len(t.value) - - def t_NUMBER(self,t): - r'\d+\.?\d*' - if t.value.find(".") != -1: - t.value = float(t.value) - else: - t.value = int(t.value) - return t - - def t_STRING(self,t): - r"'([^\\']|\\.)*'" - #t.value = t.value[1:-1].replace("\\'", "'") - t.value = t.value[1:-1] - return t - - reserved = { - 'network' : 'NETWORK', - 'ansible' : 'ANSIBLE', - 'system' : 'SYSTEM', - 'soft' : 'SOFT', - 'and' : 'AND', - 'contains' : 'CONTAINS', - 'deploy' : 'DEPLOY', - 'configure': 'CONFIGURE', - 'contextualize': 'CONTEXTUALIZE', - 'step':'STEP', - 'with':'WITH' - } - - def t_VAR(self, t): - r'[a-zA-Z_.][\w\d_.]*' - t.type = self.reserved.get(t.value, 'VAR') # Check reserved words - return t - - def t_RECIPE_BEGIN(self, t): - r'@begin' - t.lexer.push_state('recipe') - return t - - def t_recipe_RECIPE_END(self, t): - r'@end' - t.lexer.pop_state() - return t - - def t_recipe_RECIPE_LINE(self, t): - r'.*\n' - t.type = 'RECIPE_LINE' - t.lexer.lineno += t.value.count("\n") - return t - - # Error handling rule - def t_ANY_error(self, t): - #print "Illegal character '%s'" % t.value[0] - t.lexer.skip(1) - - def p_radl(self, t): - """radl : radl radl_sentence - | radl_sentence""" - - if len(t) == 2: - t[0] = RADL() - t[0].add(t[1]) - else: - t[0] = t[1] - t[0].add(t[2]) - - def p_radl_sentence(self, t): - """radl_sentence : network_sentence - | ansible_sentence - | system_sentence - | configure_sentence - | contextualize_sentence - | deploy_sentence""" - t[0] = t[1] - - def p_configure_sentence(self, t): - """configure_sentence : CONFIGURE VAR - | CONFIGURE VAR LPAREN RECIPE_BEGIN recipe RECIPE_END RPAREN""" - - if len(t) == 3: - t[0] = configure(t[2], reference=True, line=t.lineno(1)) - else: - t[0] = configure(t[2], t[5], line=t.lineno(1)) - - def p_recipe(self, t): - """recipe : RECIPE_LINE - | RECIPE_LINE recipe""" - if len(t) == 3: - t[0] = t[1] + t[2] - else: - t[0] = t[1] - - def p_deploy_sentence(self, t): - """deploy_sentence : DEPLOY VAR NUMBER - | DEPLOY VAR NUMBER VAR""" - - if len(t) == 4: - t[0] = deploy(t[2], t[3], line=t.lineno(1)) - else: - t[0] = deploy(t[2], t[3], t[4], line=t.lineno(1)) - - def p_contextualize_sentence(self, t): - """contextualize_sentence : CONTEXTUALIZE LPAREN contextualize_items RPAREN - | CONTEXTUALIZE NUMBER LPAREN contextualize_items RPAREN""" - - if len(t) == 5: - t[0] = contextualize(t[3], line=t.lineno(1)) - else: - t[0] = contextualize(t[4], t[2], line=t.lineno(1)) - - def p_contextualize_items(self, t): - """contextualize_items : contextualize_items contextualize_item - | contextualize_item - | empty""" - if len(t) == 3: - t[0] = t[1] - t[0].append(t[2]) - elif t[1]: - t[0] = [t[1]] - else: - t[0] = [] - - def p_contextualize_item(self, t): - """contextualize_item : SYSTEM VAR CONFIGURE VAR - | SYSTEM VAR CONFIGURE VAR STEP NUMBER - | SYSTEM VAR CONFIGURE VAR WITH VAR""" - - if len(t) == 5: - t[0] = contextualize_item(t[2], t[4], line=t.lineno(1)) - elif t[5] == "with": - t[0] = contextualize_item(t[2], t[4], ctxt_tool=t[6], line=t.lineno(1)) - else: - t[0] = contextualize_item(t[2], t[4], num=t[6], line=t.lineno(1)) - - def p_network_sentence(self, t): - """network_sentence : NETWORK VAR - | NETWORK VAR LPAREN features RPAREN""" - - if len(t) == 3: - t[0] = network(t[2], reference=True, line=t.lineno(1)) - else: - t[0] = network(t[2], t[4], line=t.lineno(1)) - - def p_ansible_sentence(self, t): - """ansible_sentence : ANSIBLE VAR LPAREN features RPAREN""" - - t[0] = ansible(t[2], t[4], line=t.lineno(1)) - - def p_system_sentence(self, t): - """system_sentence : SYSTEM VAR - | SYSTEM VAR LPAREN features RPAREN""" - - if len(t) == 3: - t[0] = system(t[2], reference=True, line=t.lineno(1)) - else: - t[0] = system(t[2], t[4], line=t.lineno(1)) - - - def p_features(self, t): - """features : features AND feature - | feature - | empty""" - - if len(t) == 4: - t[0] = t[1] - t[0].append(t[3]) - elif t[1]: - t[0] = [t[1]] - else: - t[0] = [] - - def p_feature(self, t): - """feature : feature_soft - | feature_simple - | feature_contains""" - - t[0] = t[1] - - def p_feature_soft(self, t): - """feature_soft : SOFT NUMBER LPAREN features RPAREN""" - - t[0] = SoftFeatures(t[2], t[4], line=t.lineno(1)) - - def p_feature_simple(self, t): - """feature_simple : VAR comparator NUMBER VAR - | VAR comparator NUMBER - | VAR comparator LBRACK string_list RBRACK - | VAR comparator STRING""" - - if len(t) == 6: - t[0] = Feature(t[1], t[2], t[4], line=t.lineno(1)) - elif len(t) == 5: - t[0] = Feature(t[1], t[2], t[3], unit=t[4], - line=t.lineno(1)) - elif len(t) == 4: - t[0] = Feature(t[1], t[2], t[3], line=t.lineno(1)) - - def p_empty(self, t): - """empty :""" - - t[0] = None - - def p_comparator(self, t): - """comparator : EQ - | LT - | GT - | GE - | LE""" - - t[0] = t[1] - - def p_feature_contains(self, t): - """feature_contains : VAR CONTAINS LPAREN features RPAREN""" - - t[0] = Feature(t[1], t[2], Features(t[4]), line=t.lineno(1)) - - def p_string_list(self, t): - """string_list : string_list COMMA STRING - | STRING - | empty""" - - if len(t) == 4: - t[0] = t[1] - t[0].append(t[3]) - elif t[1]: - t[0] = [t[1]] - else: - t[0] = [] - - def p_error(self, t): - raise RADLParseException("Parse error in: " + str(t), line=t.lineno if t else None) - - def parse(self, data): - self.lexer.lineno = 1 - self.lexer.begin('INITIAL') - return self.yacc.parse(data, tracking=True, debug=0, lexer=self.lexer) - -def parse_radl(data): - """ - Parse a RADL document. - - Args: - - data(str): filepath to a RADL content or a string with content. - - Return: RADL object. - """ - - if data is None: - return None - elif os.path.isfile(data): - f = open(data) - data = "".join(f.readlines()) - f.close() - elif data.strip() == "": - return RADL() - data = data + "\n" - - parser = RADLParser(lextab = 'radl') - return parser.parse(data) \ No newline at end of file diff --git a/setup.py b/setup.py index dcf59b3..18bb24f 100644 --- a/setup.py +++ b/setup.py @@ -31,5 +31,5 @@ long_description="IM is a tool that ease the access and the usability of IaaS clouds by automating the VMI selection, deployment, configuration, software installation, monitoring and update of Virtual Appliances. It supports APIs from a large number of virtual platforms, making user applications cloud-agnostic. In addition it integrates a contextualization system to enable the installation and configuration of all the user required applications providing the user with a fully functional infrastructure.", description="IM is a tool to manage virtual infrastructures on Cloud deployments", platforms=["any"], - install_requires=["ply","netaddr"] + install_requires=["radl","netaddr"] )