Backport v4 filters

docs/CHANGELOG.rst

@@ -1,3 +1,14 @@
+2018-01-03
+
+    Release 3.3.1
+
+    * Add new Jinja2 custom template filters to multi_sort to sort and
+      unique_together to compute unique lists. Both filter take an attributes
+      list of attribute names and use all these attribute names to sort or
+      compute unique values.
+    * Use saneyaml library to dump and load YAML
+
+
 2018-11-15
 
     Release 3.3.0

setup.py

@@ -24,7 +24,7 @@ def read(*names, **kwargs):
 
 setup(
     name='aboutcode-toolkit',
-    version='3.3.0',
+    version='3.3.1',
     license='Apache-2.0',
     description=(
         'AboutCode-toolkit is a tool to document the provenance (origin and license) of '
@@ -70,7 +70,8 @@ def read(*names, **kwargs):
         'jinja2 >= 2.9, < 3.0',
         'click >= 6.7, < 7.0',
         "backports.csv ; python_version<'3.6'",
-        'PyYAML >= 3.0, < 4.0',
+        'PyYAML >= 3.11, <=3.13',
+        'saneyaml',
         'boolean.py >= 3.5, < 4.0',
         'license_expression >= 0.94, < 1.0',
     ],

src/attributecode/__init__.py

@@ -27,7 +27,7 @@
     basestring = str  # Python 3 #NOQA
 
 
-__version__ = '3.3.0'
+__version__ = '3.3.1'
 
 
 __about_spec_version__ = '3.1'

src/attributecode/attrib.py

@@ -32,6 +32,7 @@
 import attributecode
 from attributecode import ERROR
 from attributecode import Error
+from attributecode.attrib_util import get_template
 from attributecode.licenses import COMMON_LICENSES
 from attributecode.model import parse_license_expression
 from attributecode.util import add_unc
@@ -46,7 +47,7 @@ def generate(abouts, template_string=None, vartext_dict=None):
     syntax_error = check_template(template_string)
     if syntax_error:
         return 'Template validation error at line: %r: %r' % (syntax_error)
-    template = jinja2.Template(template_string)
+    template = get_template(template_string)
 
     try:
         captured_license = []
@@ -119,13 +120,14 @@ def generate(abouts, template_string=None, vartext_dict=None):
     return rendered
 
 
+
 def check_template(template_string):
     """
     Check the syntax of a template. Return an error tuple (line number,
     message) if the template is invalid or None if it is valid.
     """
     try:
-        jinja2.Template(template_string)
+        get_template(template_string)
     except (jinja2.TemplateSyntaxError, jinja2.TemplateAssertionError) as e:
         return e.lineno, e.message
 

src/attributecode/attrib_util.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: utf8 -*-
+
+# ============================================================================
+#  Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+# ============================================================================
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from jinja2 import Environment
+from jinja2.filters import environmentfilter
+from jinja2.filters import make_attrgetter
+from jinja2.filters import ignore_case
+from jinja2.filters import FilterArgumentError
+
+
+"""
+Extra JINJA2 custom filters and other template utilities.
+"""
+
+
+def get_template(template_text):
+    """
+    Return a template built from a text string.
+    Register custom templates as needed.
+    """
+    env = Environment(autoescape=True)
+    # register our custom filters
+    env.filters.update(dict(
+        unique_together=unique_together,
+        multi_sort=multi_sort))
+    return env.from_string(template_text)
+
+
+@environmentfilter
+def multi_sort(environment, value, reverse=False, case_sensitive=False,
+               attributes=None):
+    """
+    Sort an iterable using an "attributes" list of attribute names available on
+    each iterable item. Sort ascending unless reverse is "true". Ignore the case
+    of strings unless "case_sensitive" is "true".
+
+    .. sourcecode:: jinja
+
+        {% for item in iterable|multi_sort(attributes=['date', 'name']) %}
+            ...
+        {% endfor %}
+    """
+    if not attributes:
+        raise FilterArgumentError(
+            'The multi_sort filter requires a list of attributes as argument, '
+            'such as in: '
+            "for item in iterable|multi_sort(attributes=['date', 'name'])")
+
+    # build a list of attribute getters, one for each attribute
+    do_ignore_case = ignore_case if not case_sensitive else None
+    attribute_getters = []
+    for attribute in attributes:
+        ag = make_attrgetter(environment, attribute, postprocess=do_ignore_case)
+        attribute_getters.append(ag)
+
+    # build a key function that has runs all attribute getters
+    def key(v):
+        return [a(v) for a in attribute_getters]
+
+    return sorted(value, key=key, reverse=reverse)
+
+
+@environmentfilter
+def unique_together(environment, value, case_sensitive=False, attributes=None):
+    """
+    Return a list of unique items from an iterable. Unicity is checked when
+    considering together all the values of an "attributes" list of attribute
+    names available on each iterable item.. The items order is preserved. Ignore
+    the case of strings unless "case_sensitive" is "true".
+    .. sourcecode:: jinja
+
+        {% for item in iterable|unique_together(attributes=['date', 'name']) %}
+            ...
+        {% endfor %}
+
+    """
+    if not attributes:
+        raise FilterArgumentError(
+            'The unique_together filter requires a list of attributes as argument, '
+            'such as in: '
+            "{% for item in iterable|unique_together(attributes=['date', 'name']) %} ")
+
+    # build a list of attribute getters, one for each attribute
+    do_ignore_case = ignore_case if not case_sensitive else None
+    attribute_getters = []
+    for attribute in attributes:
+        ag = make_attrgetter(environment, attribute, postprocess=do_ignore_case)
+        attribute_getters.append(ag)
+
+    # build a unique_key function that has runs all attribute getters
+    # and returns a hashable tuple
+    def unique_key(v):
+        return tuple(repr(a(v)) for a in attribute_getters)
+
+    unique = []
+    seen = set()
+    for item in value:
+        key = unique_key(item)
+        if key not in seen:
+            seen.add(key)
+            unique.append(item)
+    return unique

src/attributecode/model.py

@@ -53,14 +53,14 @@
     from urllib.error import HTTPError  # NOQA
 
 from license_expression import Licensing
+import saneyaml
 
 from attributecode import CRITICAL
 from attributecode import ERROR
 from attributecode import INFO
 from attributecode import WARNING
 from attributecode import api
 from attributecode import Error
-from attributecode import saneyaml
 from attributecode import util
 from attributecode.util import add_unc
 from attributecode.util import copy_license_notice_files
@@ -1015,8 +1015,6 @@ def load(self, location, use_mapping=False, mapping_file=None):
             loc = add_unc(loc)
             with codecs.open(loc, encoding='utf-8') as txt:
                 input_text = txt.read()
-            # Check for duplicated key
-            yaml.load(input_text, Loader=util.NoDuplicateLoader)
             """
             The running_inventory defines if the current process is 'inventory' or not.
             This is used for the validation of the path of the 'about_resource'.
@@ -1030,7 +1028,9 @@ def load(self, location, use_mapping=False, mapping_file=None):
             # wrap the value of the boolean field in quote to avoid
             # automatically conversion from yaml.load
             input = util.wrap_boolean_value(input_text)  # NOQA
-            errs = self.load_dict(saneyaml.load(input), base_dir, running_inventory, use_mapping, mapping_file)
+            data = saneyaml.load(input, allow_duplicate_keys=False)
+
+            errs = self.load_dict(data, base_dir, running_inventory, use_mapping, mapping_file)
             errors.extend(errs)
         except Exception as e:
             msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n' + str(e)
@@ -1081,7 +1081,7 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e
         If with_absent, include absent (not present) fields.
         If with_empty, include empty fields.
         """
-        about_data = {}
+        about_data = OrderedDict()
         # Group the same license information (name, url, file) together
         license_key = []
         license_name = []
@@ -1110,7 +1110,7 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e
         # Group the same license information in a list
         license_group = list(zip_longest(license_key, license_name, license_file, license_url))
         for lic_group in license_group:
-            lic_dict = {}
+            lic_dict = OrderedDict()
             if lic_group[0]:
                 lic_dict['key'] = lic_group[0]
             if lic_group[1]:
@@ -1121,7 +1121,8 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e
                 lic_dict['url'] = lic_group[3]
             about_data.setdefault('licenses', []).append(lic_dict)
         formatted_about_data = util.format_output(about_data, use_mapping, mapping_file)
-        return saneyaml.dump(formatted_about_data)
+
+        return saneyaml.dump(formatted_about_data, indent=2)
 
     def dump(self, location, use_mapping=False, mapping_file=False, with_absent=False, with_empty=True):
         """