diff --git a/docs/CHANGELOG.rst b/docs/CHANGELOG.rst index 8a987e6c..6a487e56 100644 --- a/docs/CHANGELOG.rst +++ b/docs/CHANGELOG.rst @@ -1,3 +1,14 @@ +2018-01-03 + + Release 3.3.1 + + * Add new Jinja2 custom template filters to multi_sort to sort and + unique_together to compute unique lists. Both filter take an attributes + list of attribute names and use all these attribute names to sort or + compute unique values. + * Use saneyaml library to dump and load YAML + + 2018-11-15 Release 3.3.0 diff --git a/setup.py b/setup.py index 33edfaa7..55c1fcfb 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def read(*names, **kwargs): setup( name='aboutcode-toolkit', - version='3.3.0', + version='3.3.1', license='Apache-2.0', description=( 'AboutCode-toolkit is a tool to document the provenance (origin and license) of ' @@ -70,7 +70,8 @@ def read(*names, **kwargs): 'jinja2 >= 2.9, < 3.0', 'click >= 6.7, < 7.0', "backports.csv ; python_version<'3.6'", - 'PyYAML >= 3.0, < 4.0', + 'PyYAML >= 3.11, <=3.13', + 'saneyaml', 'boolean.py >= 3.5, < 4.0', 'license_expression >= 0.94, < 1.0', ], diff --git a/src/attributecode/__init__.py b/src/attributecode/__init__.py index 0397bfa8..56941c3b 100644 --- a/src/attributecode/__init__.py +++ b/src/attributecode/__init__.py @@ -27,7 +27,7 @@ basestring = str # Python 3 #NOQA -__version__ = '3.3.0' +__version__ = '3.3.1' __about_spec_version__ = '3.1' diff --git a/src/attributecode/attrib.py b/src/attributecode/attrib.py index 820123ad..6d1545d5 100644 --- a/src/attributecode/attrib.py +++ b/src/attributecode/attrib.py @@ -32,6 +32,7 @@ import attributecode from attributecode import ERROR from attributecode import Error +from attributecode.attrib_util import get_template from attributecode.licenses import COMMON_LICENSES from attributecode.model import parse_license_expression from attributecode.util import add_unc @@ -46,7 +47,7 @@ def generate(abouts, template_string=None, vartext_dict=None): syntax_error = check_template(template_string) if syntax_error: return 'Template validation error at line: %r: %r' % (syntax_error) - template = jinja2.Template(template_string) + template = get_template(template_string) try: captured_license = [] @@ -119,13 +120,14 @@ def generate(abouts, template_string=None, vartext_dict=None): return rendered + def check_template(template_string): """ Check the syntax of a template. Return an error tuple (line number, message) if the template is invalid or None if it is valid. """ try: - jinja2.Template(template_string) + get_template(template_string) except (jinja2.TemplateSyntaxError, jinja2.TemplateAssertionError) as e: return e.lineno, e.message diff --git a/src/attributecode/attrib_util.py b/src/attributecode/attrib_util.py new file mode 100644 index 00000000..cf665905 --- /dev/null +++ b/src/attributecode/attrib_util.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf8 -*- + +# ============================================================================ +# Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from jinja2 import Environment +from jinja2.filters import environmentfilter +from jinja2.filters import make_attrgetter +from jinja2.filters import ignore_case +from jinja2.filters import FilterArgumentError + + +""" +Extra JINJA2 custom filters and other template utilities. +""" + + +def get_template(template_text): + """ + Return a template built from a text string. + Register custom templates as needed. + """ + env = Environment(autoescape=True) + # register our custom filters + env.filters.update(dict( + unique_together=unique_together, + multi_sort=multi_sort)) + return env.from_string(template_text) + + +@environmentfilter +def multi_sort(environment, value, reverse=False, case_sensitive=False, + attributes=None): + """ + Sort an iterable using an "attributes" list of attribute names available on + each iterable item. Sort ascending unless reverse is "true". Ignore the case + of strings unless "case_sensitive" is "true". + + .. sourcecode:: jinja + + {% for item in iterable|multi_sort(attributes=['date', 'name']) %} + ... + {% endfor %} + """ + if not attributes: + raise FilterArgumentError( + 'The multi_sort filter requires a list of attributes as argument, ' + 'such as in: ' + "for item in iterable|multi_sort(attributes=['date', 'name'])") + + # build a list of attribute getters, one for each attribute + do_ignore_case = ignore_case if not case_sensitive else None + attribute_getters = [] + for attribute in attributes: + ag = make_attrgetter(environment, attribute, postprocess=do_ignore_case) + attribute_getters.append(ag) + + # build a key function that has runs all attribute getters + def key(v): + return [a(v) for a in attribute_getters] + + return sorted(value, key=key, reverse=reverse) + + +@environmentfilter +def unique_together(environment, value, case_sensitive=False, attributes=None): + """ + Return a list of unique items from an iterable. Unicity is checked when + considering together all the values of an "attributes" list of attribute + names available on each iterable item.. The items order is preserved. Ignore + the case of strings unless "case_sensitive" is "true". + .. sourcecode:: jinja + + {% for item in iterable|unique_together(attributes=['date', 'name']) %} + ... + {% endfor %} + + """ + if not attributes: + raise FilterArgumentError( + 'The unique_together filter requires a list of attributes as argument, ' + 'such as in: ' + "{% for item in iterable|unique_together(attributes=['date', 'name']) %} ") + + # build a list of attribute getters, one for each attribute + do_ignore_case = ignore_case if not case_sensitive else None + attribute_getters = [] + for attribute in attributes: + ag = make_attrgetter(environment, attribute, postprocess=do_ignore_case) + attribute_getters.append(ag) + + # build a unique_key function that has runs all attribute getters + # and returns a hashable tuple + def unique_key(v): + return tuple(repr(a(v)) for a in attribute_getters) + + unique = [] + seen = set() + for item in value: + key = unique_key(item) + if key not in seen: + seen.add(key) + unique.append(item) + return unique diff --git a/src/attributecode/model.py b/src/attributecode/model.py index 0e6ee4ed..ad81f1ff 100644 --- a/src/attributecode/model.py +++ b/src/attributecode/model.py @@ -53,6 +53,7 @@ from urllib.error import HTTPError # NOQA from license_expression import Licensing +import saneyaml from attributecode import CRITICAL from attributecode import ERROR @@ -60,7 +61,6 @@ from attributecode import WARNING from attributecode import api from attributecode import Error -from attributecode import saneyaml from attributecode import util from attributecode.util import add_unc from attributecode.util import copy_license_notice_files @@ -1015,8 +1015,6 @@ def load(self, location, use_mapping=False, mapping_file=None): loc = add_unc(loc) with codecs.open(loc, encoding='utf-8') as txt: input_text = txt.read() - # Check for duplicated key - yaml.load(input_text, Loader=util.NoDuplicateLoader) """ The running_inventory defines if the current process is 'inventory' or not. This is used for the validation of the path of the 'about_resource'. @@ -1030,7 +1028,9 @@ def load(self, location, use_mapping=False, mapping_file=None): # wrap the value of the boolean field in quote to avoid # automatically conversion from yaml.load input = util.wrap_boolean_value(input_text) # NOQA - errs = self.load_dict(saneyaml.load(input), base_dir, running_inventory, use_mapping, mapping_file) + data = saneyaml.load(input, allow_duplicate_keys=False) + + errs = self.load_dict(data, base_dir, running_inventory, use_mapping, mapping_file) errors.extend(errs) except Exception as e: msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n' + str(e) @@ -1081,7 +1081,7 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e If with_absent, include absent (not present) fields. If with_empty, include empty fields. """ - about_data = {} + about_data = OrderedDict() # Group the same license information (name, url, file) together license_key = [] license_name = [] @@ -1110,7 +1110,7 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e # Group the same license information in a list license_group = list(zip_longest(license_key, license_name, license_file, license_url)) for lic_group in license_group: - lic_dict = {} + lic_dict = OrderedDict() if lic_group[0]: lic_dict['key'] = lic_group[0] if lic_group[1]: @@ -1121,7 +1121,8 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e lic_dict['url'] = lic_group[3] about_data.setdefault('licenses', []).append(lic_dict) formatted_about_data = util.format_output(about_data, use_mapping, mapping_file) - return saneyaml.dump(formatted_about_data) + + return saneyaml.dump(formatted_about_data, indent=2) def dump(self, location, use_mapping=False, mapping_file=False, with_absent=False, with_empty=True): """ diff --git a/src/attributecode/saneyaml.py b/src/attributecode/saneyaml.py deleted file mode 100644 index cbe5db37..00000000 --- a/src/attributecode/saneyaml.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf8 -*- - -# ============================================================================ -# Copyright (c) 2015-2017 nexB Inc. http://www.nexb.com/ - All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from collections import OrderedDict -from functools import partial - -import yaml -import sys - -try: - from yaml import CSafeLoader as SafeLoader - from yaml import CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeLoader - from yaml import SafeDumper - -try: - unicode # Python 2 -except NameError: - unicode = str # Python 3 #NOQA - -try: - basestring # Python 2 -except NameError: - basestring = str # Python 3 #NOQA - - -""" -Wrapper around PyYAML to provide sane defaults ensuring that dump/load does -not damage content, keeps ordering, use always block-style and use four -spaces indents to get readable YAML and quotes and folds texts in a sane way. - -Use the `load` function to get a primitive type from a YAML string and the -`dump` function to get a YAML string from a primitive type. - -Load and dump rely on subclasses of SafeLoader and SafeDumper respectively -doing all the dirty bidding to get PyYAML straight. -""" - -# Check: -# https://github.com/ralienpp/reyaml/blob/master/reyaml/__init__.py -# https://pypi.python.org/pypi/PyYAML.Yandex/3.11.1 -# https://pypi.python.org/pypi/ruamel.yaml/0.9.1 -# https://pypi.python.org/pypi/yaml2rst/0.2 - -def load(s): - """ - Return an object safely loaded from YAML string `s`. `s` must be unicode - or be a string that converts to unicode without errors. - """ - return yaml.load(s, Loader=SaneLoader) - - -def dump(obj): - """ - Return a safe YAML unicode string representation from `obj`. - """ - kwargs = dict( - Dumper=SaneDumper, - default_flow_style=False, - default_style=None, - canonical=False, - allow_unicode=True, - # do not encode Unicode - encoding=None, - indent=4, - width=80, - line_break='\n', - explicit_start=False, - explicit_end=False, - ) - return yaml.dump(obj, **kwargs) - - -class SaneLoader(SafeLoader): - pass - - -def string_loader(loader, node): - """ - Ensure that a scalar type (a value) is returned as a plain unicode string. - """ - return loader.construct_scalar(node) - - -SaneLoader.add_constructor(u'tag:yaml.org,2002:str', string_loader) - - -# Load as strings most scalar types: nulls, booleans, ints, (such as in -# version 01) floats (such version 2.20) and timestamps conversion (in -# versions too) are all emitted as unicode strings. This avoid unwanted type -# conversions for unquoted strings and the resulting content damaging. This -# overrides the implicit resolvers. Callers must handle type conversion -# explicitly from unicode to other types in the loaded objects. - -SaneLoader.add_constructor(u'tag:yaml.org,2002:null', string_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:boolean', string_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:timestamp', string_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:float', string_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:int', string_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:null', string_loader) - - -def ordered_loader(loader, node): - """ - Ensure that YAML maps ordered is preserved and loaded in an OrderedDict. - """ - assert isinstance(node, yaml.MappingNode) - omap = OrderedDict() - yield omap - - for key, value in node.value: - key = loader.construct_object(key) - value = loader.construct_object(value) - omap[key] = value - -SaneLoader.add_constructor(u'tag:yaml.org,2002:map', ordered_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:omap', ordered_loader) - - -class SaneDumper(SafeDumper): - """ - Ensure that lists items are always indented. - """ - def increase_indent(self, flow=False, indentless=False): # @UnusedVariable - return super(SaneDumper, self).increase_indent(flow, indentless=False) - - -def ordered_dumper(dumper, data): - """ - Ensure that maps are always dumped in the items order. - """ - return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.items()) - -SaneDumper.add_representer(OrderedDict, ordered_dumper) - - -def null_dumper(dumper, value): # @UnusedVariable - """ - Always dump nulls as empty string. - """ - return dumper.represent_scalar(u'tag:yaml.org,2002:null', u'') - -SafeDumper.add_representer(type(None), null_dumper) - - -def string_dumper(dumper, value, _tag=u'tag:yaml.org,2002:str'): - """ - Ensure that all scalars are dumped as UTF-8 unicode, folded and quoted in - the sanest and most readable way. - """ - style = None - - if not isinstance(value, basestring): - value = repr(value) - - if isinstance(value, str): - if sys.version_info[0] < 3: # Python 2 - value = value.decode('utf-8') - - folded_style = '>' - verbatim_style = '|' -# single_style = "'" -# double_style = '"' - - long_lines = any(len(line) > 40 for line in value.splitlines(False)) and ' ' in value - multilines = '\n' in value -# single_quote = "'" in value -# double_quote = '"' in value -# colon_space = ': ' in value -# hash_space = '# ' in value - - if multilines: # or colon_space or hash_space or (single_quote and double_quote) or double_quote: - style = verbatim_style - elif long_lines: - style = folded_style -# elif single_quote and double_quote: -# style = folded_style -# elif single_quote: -# style = double_style -# elif double_quote: -# style = single_style - - return dumper.represent_scalar(_tag, value, style=style) - -SaneDumper.add_representer(str, string_dumper) -SaneDumper.add_representer(unicode, string_dumper) -SaneDumper.add_representer(int, partial(string_dumper, _tag=u'tag:yaml.org,2002:int')) -SaneDumper.add_representer(float, partial(string_dumper, _tag=u'tag:yaml.org,2002:float')) - - -def boolean_dumper(dumper, value): - """ - Dump booleans as yes or no. - """ - value = u'yes' if value else u'no' - style = None - return dumper.represent_scalar(u'tag:yaml.org,2002:bool', value, style=style) - -SaneDumper.add_representer(bool, boolean_dumper) diff --git a/tests/test_gen.py b/tests/test_gen.py index 5ece6d76..f7a3423d 100644 --- a/tests/test_gen.py +++ b/tests/test_gen.py @@ -70,12 +70,10 @@ def test_load_inventory(self): else: assert e.message in expected_error_messages - expected = [u'about_resource: .\n' - u'name: AboutCode\n' - u'version: 0.11.0\n' - u'description: |-\n' - u' multi\n' - u' line\n'] + expected = ['about_resource: .\n' + 'name: AboutCode\n' + 'version: 0.11.0\n' + 'description: |\n multi\n line\n'] result = [a.dumps(use_mapping=False, mapping_file=False, with_absent=False, with_empty=False) for a in abouts] assert expected == result @@ -101,15 +99,14 @@ def test_load_inventory_with_mapping(self): else: assert e.message in expected_error_messages - expected = [u'about_resource: .\n' - u'name: AboutCode\n' - u'version: 0.11.0\n' - u'copyright: Copyright (c) nexB, Inc.\n' - u'resource: this.ABOUT\n' - u'description: |-\n' - u' multi\n' - u' line\n' - ] + expected = [ + 'about_resource: .\n' + 'name: AboutCode\n' + 'version: 0.11.0\n' + 'copyright: Copyright (c) nexB, Inc.\n' + 'resource: this.ABOUT\n' + 'description: |\n multi\n line\n' + ] result = [a.dumps(use_mapping, mapping_file=False, with_absent=False, with_empty=False) for a in abouts] assert expected == result @@ -171,9 +168,9 @@ def test_generate(self): expected = (u'about_resource: .\n' u'name: AboutCode\n' u'version: 0.11.0\n' - u'description: |-\n' - u' multi\n' - u' line\n') + u'description: |\n' + u' multi\n' + u' line\n') assert expected == in_mem_result def test_generate_not_overwrite_original_license_file(self): @@ -190,7 +187,7 @@ def test_generate_not_overwrite_original_license_file(self): u'name: AboutCode\n' u'version: 0.11.0\n' u'licenses:\n' - u' - file: this.LICENSE\n') + u' - file: this.LICENSE\n') assert expected == in_mem_result def test_deduplicate(self): diff --git a/tests/test_model.py b/tests/test_model.py index daf15738..c46f7ed5 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -709,24 +709,24 @@ def test_About_dumps(self): a = model.About(test_file) assert [] == a.errors - expected = u'''about_resource: . + expected = '''about_resource: . name: AboutCode version: 0.11.0 copyright: Copyright (c) 2013-2014 nexB Inc. license_expression: apache-2.0 author: - - Jillian Daguil - - Chin Yeung Li - - Philippe Ombredanne - - Thomas Druez -description: |- - AboutCode is a tool - to process ABOUT files. - An ABOUT file is a file. + - Jillian Daguil + - Chin Yeung Li + - Philippe Ombredanne + - Thomas Druez +description: | + AboutCode is a tool + to process ABOUT files. + An ABOUT file is a file. homepage_url: http://dejacode.org licenses: - - file: apache-2.0.LICENSE - key: apache-2.0 + - key: apache-2.0 + file: apache-2.0.LICENSE notice_file: NOTICE owner: nexB Inc. vcs_repository: https://github.com/dejacode/about-code-tool.git diff --git a/thirdparty/saneyaml-0.1-py2.py3-none-any.whl b/thirdparty/saneyaml-0.1-py2.py3-none-any.whl new file mode 100644 index 00000000..c8aa66fa Binary files /dev/null and b/thirdparty/saneyaml-0.1-py2.py3-none-any.whl differ diff --git a/thirdparty/saneyaml-0.1-py2.py3-none-any.whl.ABOUT b/thirdparty/saneyaml-0.1-py2.py3-none-any.whl.ABOUT new file mode 100644 index 00000000..ae857b3e --- /dev/null +++ b/thirdparty/saneyaml-0.1-py2.py3-none-any.whl.ABOUT @@ -0,0 +1,20 @@ +about_resource: saneyaml-0.1-py2.py3-none-any.whl +attribute: true +checksum_md5: 53509e4f1ee9f6565158163a56513b7c +checksum_sha1: a66920309794ead47711473a21f1c7d003acd005 +contact: http://www.nexb.com/contactus.html +copyright: Copyright (c) nexB Inc. and others. +download_url: https://files.pythonhosted.org/packages/21/13/3d639adcd97fc22cfc4acfd23330cc26f96a85a5b992d93086684e24ac14/saneyaml-0.1-py2.py3-none-any.whl +homepage_url: https://github.com/nexB/saneyaml +license_expression: apache-2.0 +licenses: +- file: apache-2.0.LICENSE + key: apache-2.0 + name: Apache License 2.0 +name: saneyaml +owner: nexB +owner_url: http://www.nexb.com/ +package_url: pkg:pypi/saneyaml@0.1 +track_changes: true +vcs_repository: https://github.com/nexB/saneyaml +version: '0.1'