From a4e31f149b5b1bceafe89b51955ef1b57a90008f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 3 Jan 2019 19:57:18 +0100 Subject: [PATCH 1/4] Backport Jinja filters from v4 branch Signed-off-by: Philippe Ombredanne --- src/attributecode/attrib.py | 6 +- src/attributecode/attrib_util.py | 119 +++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 src/attributecode/attrib_util.py diff --git a/src/attributecode/attrib.py b/src/attributecode/attrib.py index 820123ad..6d1545d5 100644 --- a/src/attributecode/attrib.py +++ b/src/attributecode/attrib.py @@ -32,6 +32,7 @@ import attributecode from attributecode import ERROR from attributecode import Error +from attributecode.attrib_util import get_template from attributecode.licenses import COMMON_LICENSES from attributecode.model import parse_license_expression from attributecode.util import add_unc @@ -46,7 +47,7 @@ def generate(abouts, template_string=None, vartext_dict=None): syntax_error = check_template(template_string) if syntax_error: return 'Template validation error at line: %r: %r' % (syntax_error) - template = jinja2.Template(template_string) + template = get_template(template_string) try: captured_license = [] @@ -119,13 +120,14 @@ def generate(abouts, template_string=None, vartext_dict=None): return rendered + def check_template(template_string): """ Check the syntax of a template. Return an error tuple (line number, message) if the template is invalid or None if it is valid. """ try: - jinja2.Template(template_string) + get_template(template_string) except (jinja2.TemplateSyntaxError, jinja2.TemplateAssertionError) as e: return e.lineno, e.message diff --git a/src/attributecode/attrib_util.py b/src/attributecode/attrib_util.py new file mode 100644 index 00000000..cf665905 --- /dev/null +++ b/src/attributecode/attrib_util.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf8 -*- + +# ============================================================================ +# Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from jinja2 import Environment +from jinja2.filters import environmentfilter +from jinja2.filters import make_attrgetter +from jinja2.filters import ignore_case +from jinja2.filters import FilterArgumentError + + +""" +Extra JINJA2 custom filters and other template utilities. +""" + + +def get_template(template_text): + """ + Return a template built from a text string. + Register custom templates as needed. + """ + env = Environment(autoescape=True) + # register our custom filters + env.filters.update(dict( + unique_together=unique_together, + multi_sort=multi_sort)) + return env.from_string(template_text) + + +@environmentfilter +def multi_sort(environment, value, reverse=False, case_sensitive=False, + attributes=None): + """ + Sort an iterable using an "attributes" list of attribute names available on + each iterable item. Sort ascending unless reverse is "true". Ignore the case + of strings unless "case_sensitive" is "true". + + .. sourcecode:: jinja + + {% for item in iterable|multi_sort(attributes=['date', 'name']) %} + ... + {% endfor %} + """ + if not attributes: + raise FilterArgumentError( + 'The multi_sort filter requires a list of attributes as argument, ' + 'such as in: ' + "for item in iterable|multi_sort(attributes=['date', 'name'])") + + # build a list of attribute getters, one for each attribute + do_ignore_case = ignore_case if not case_sensitive else None + attribute_getters = [] + for attribute in attributes: + ag = make_attrgetter(environment, attribute, postprocess=do_ignore_case) + attribute_getters.append(ag) + + # build a key function that has runs all attribute getters + def key(v): + return [a(v) for a in attribute_getters] + + return sorted(value, key=key, reverse=reverse) + + +@environmentfilter +def unique_together(environment, value, case_sensitive=False, attributes=None): + """ + Return a list of unique items from an iterable. Unicity is checked when + considering together all the values of an "attributes" list of attribute + names available on each iterable item.. The items order is preserved. Ignore + the case of strings unless "case_sensitive" is "true". + .. sourcecode:: jinja + + {% for item in iterable|unique_together(attributes=['date', 'name']) %} + ... + {% endfor %} + + """ + if not attributes: + raise FilterArgumentError( + 'The unique_together filter requires a list of attributes as argument, ' + 'such as in: ' + "{% for item in iterable|unique_together(attributes=['date', 'name']) %} ") + + # build a list of attribute getters, one for each attribute + do_ignore_case = ignore_case if not case_sensitive else None + attribute_getters = [] + for attribute in attributes: + ag = make_attrgetter(environment, attribute, postprocess=do_ignore_case) + attribute_getters.append(ag) + + # build a unique_key function that has runs all attribute getters + # and returns a hashable tuple + def unique_key(v): + return tuple(repr(a(v)) for a in attribute_getters) + + unique = [] + seen = set() + for item in value: + key = unique_key(item) + if key not in seen: + seen.add(key) + unique.append(item) + return unique From 1a77f7b8e56b9dd576b2847ee43850cd597965d5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 3 Jan 2019 20:10:33 +0100 Subject: [PATCH 2/4] Use saneyaml library Signed-off-by: Philippe Ombredanne --- setup.py | 3 +- src/attributecode/model.py | 11 +- src/attributecode/saneyaml.py | 217 ------------------ tests/test_gen.py | 35 ++- tests/test_model.py | 22 +- thirdparty/saneyaml-0.1-py2.py3-none-any.whl | Bin 0 -> 5567 bytes .../saneyaml-0.1-py2.py3-none-any.whl.ABOUT | 20 ++ 7 files changed, 55 insertions(+), 253 deletions(-) delete mode 100644 src/attributecode/saneyaml.py create mode 100644 thirdparty/saneyaml-0.1-py2.py3-none-any.whl create mode 100644 thirdparty/saneyaml-0.1-py2.py3-none-any.whl.ABOUT diff --git a/setup.py b/setup.py index 33edfaa7..78dca192 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,8 @@ def read(*names, **kwargs): 'jinja2 >= 2.9, < 3.0', 'click >= 6.7, < 7.0', "backports.csv ; python_version<'3.6'", - 'PyYAML >= 3.0, < 4.0', + 'PyYAML >= 3.11, <=3.13', + 'saneyaml', 'boolean.py >= 3.5, < 4.0', 'license_expression >= 0.94, < 1.0', ], diff --git a/src/attributecode/model.py b/src/attributecode/model.py index 0e6ee4ed..016bafb7 100644 --- a/src/attributecode/model.py +++ b/src/attributecode/model.py @@ -53,6 +53,7 @@ from urllib.error import HTTPError # NOQA from license_expression import Licensing +import saneyaml from attributecode import CRITICAL from attributecode import ERROR @@ -60,7 +61,6 @@ from attributecode import WARNING from attributecode import api from attributecode import Error -from attributecode import saneyaml from attributecode import util from attributecode.util import add_unc from attributecode.util import copy_license_notice_files @@ -1015,8 +1015,6 @@ def load(self, location, use_mapping=False, mapping_file=None): loc = add_unc(loc) with codecs.open(loc, encoding='utf-8') as txt: input_text = txt.read() - # Check for duplicated key - yaml.load(input_text, Loader=util.NoDuplicateLoader) """ The running_inventory defines if the current process is 'inventory' or not. This is used for the validation of the path of the 'about_resource'. @@ -1030,7 +1028,9 @@ def load(self, location, use_mapping=False, mapping_file=None): # wrap the value of the boolean field in quote to avoid # automatically conversion from yaml.load input = util.wrap_boolean_value(input_text) # NOQA - errs = self.load_dict(saneyaml.load(input), base_dir, running_inventory, use_mapping, mapping_file) + data = saneyaml.load(input, allow_duplicate_keys=False) + + errs = self.load_dict(data, base_dir, running_inventory, use_mapping, mapping_file) errors.extend(errs) except Exception as e: msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n' + str(e) @@ -1121,7 +1121,8 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e lic_dict['url'] = lic_group[3] about_data.setdefault('licenses', []).append(lic_dict) formatted_about_data = util.format_output(about_data, use_mapping, mapping_file) - return saneyaml.dump(formatted_about_data) + + return saneyaml.dump(formatted_about_data, indent=2) def dump(self, location, use_mapping=False, mapping_file=False, with_absent=False, with_empty=True): """ diff --git a/src/attributecode/saneyaml.py b/src/attributecode/saneyaml.py deleted file mode 100644 index cbe5db37..00000000 --- a/src/attributecode/saneyaml.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf8 -*- - -# ============================================================================ -# Copyright (c) 2015-2017 nexB Inc. http://www.nexb.com/ - All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from collections import OrderedDict -from functools import partial - -import yaml -import sys - -try: - from yaml import CSafeLoader as SafeLoader - from yaml import CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeLoader - from yaml import SafeDumper - -try: - unicode # Python 2 -except NameError: - unicode = str # Python 3 #NOQA - -try: - basestring # Python 2 -except NameError: - basestring = str # Python 3 #NOQA - - -""" -Wrapper around PyYAML to provide sane defaults ensuring that dump/load does -not damage content, keeps ordering, use always block-style and use four -spaces indents to get readable YAML and quotes and folds texts in a sane way. - -Use the `load` function to get a primitive type from a YAML string and the -`dump` function to get a YAML string from a primitive type. - -Load and dump rely on subclasses of SafeLoader and SafeDumper respectively -doing all the dirty bidding to get PyYAML straight. -""" - -# Check: -# https://github.com/ralienpp/reyaml/blob/master/reyaml/__init__.py -# https://pypi.python.org/pypi/PyYAML.Yandex/3.11.1 -# https://pypi.python.org/pypi/ruamel.yaml/0.9.1 -# https://pypi.python.org/pypi/yaml2rst/0.2 - -def load(s): - """ - Return an object safely loaded from YAML string `s`. `s` must be unicode - or be a string that converts to unicode without errors. - """ - return yaml.load(s, Loader=SaneLoader) - - -def dump(obj): - """ - Return a safe YAML unicode string representation from `obj`. - """ - kwargs = dict( - Dumper=SaneDumper, - default_flow_style=False, - default_style=None, - canonical=False, - allow_unicode=True, - # do not encode Unicode - encoding=None, - indent=4, - width=80, - line_break='\n', - explicit_start=False, - explicit_end=False, - ) - return yaml.dump(obj, **kwargs) - - -class SaneLoader(SafeLoader): - pass - - -def string_loader(loader, node): - """ - Ensure that a scalar type (a value) is returned as a plain unicode string. - """ - return loader.construct_scalar(node) - - -SaneLoader.add_constructor(u'tag:yaml.org,2002:str', string_loader) - - -# Load as strings most scalar types: nulls, booleans, ints, (such as in -# version 01) floats (such version 2.20) and timestamps conversion (in -# versions too) are all emitted as unicode strings. This avoid unwanted type -# conversions for unquoted strings and the resulting content damaging. This -# overrides the implicit resolvers. Callers must handle type conversion -# explicitly from unicode to other types in the loaded objects. - -SaneLoader.add_constructor(u'tag:yaml.org,2002:null', string_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:boolean', string_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:timestamp', string_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:float', string_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:int', string_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:null', string_loader) - - -def ordered_loader(loader, node): - """ - Ensure that YAML maps ordered is preserved and loaded in an OrderedDict. - """ - assert isinstance(node, yaml.MappingNode) - omap = OrderedDict() - yield omap - - for key, value in node.value: - key = loader.construct_object(key) - value = loader.construct_object(value) - omap[key] = value - -SaneLoader.add_constructor(u'tag:yaml.org,2002:map', ordered_loader) -SaneLoader.add_constructor(u'tag:yaml.org,2002:omap', ordered_loader) - - -class SaneDumper(SafeDumper): - """ - Ensure that lists items are always indented. - """ - def increase_indent(self, flow=False, indentless=False): # @UnusedVariable - return super(SaneDumper, self).increase_indent(flow, indentless=False) - - -def ordered_dumper(dumper, data): - """ - Ensure that maps are always dumped in the items order. - """ - return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.items()) - -SaneDumper.add_representer(OrderedDict, ordered_dumper) - - -def null_dumper(dumper, value): # @UnusedVariable - """ - Always dump nulls as empty string. - """ - return dumper.represent_scalar(u'tag:yaml.org,2002:null', u'') - -SafeDumper.add_representer(type(None), null_dumper) - - -def string_dumper(dumper, value, _tag=u'tag:yaml.org,2002:str'): - """ - Ensure that all scalars are dumped as UTF-8 unicode, folded and quoted in - the sanest and most readable way. - """ - style = None - - if not isinstance(value, basestring): - value = repr(value) - - if isinstance(value, str): - if sys.version_info[0] < 3: # Python 2 - value = value.decode('utf-8') - - folded_style = '>' - verbatim_style = '|' -# single_style = "'" -# double_style = '"' - - long_lines = any(len(line) > 40 for line in value.splitlines(False)) and ' ' in value - multilines = '\n' in value -# single_quote = "'" in value -# double_quote = '"' in value -# colon_space = ': ' in value -# hash_space = '# ' in value - - if multilines: # or colon_space or hash_space or (single_quote and double_quote) or double_quote: - style = verbatim_style - elif long_lines: - style = folded_style -# elif single_quote and double_quote: -# style = folded_style -# elif single_quote: -# style = double_style -# elif double_quote: -# style = single_style - - return dumper.represent_scalar(_tag, value, style=style) - -SaneDumper.add_representer(str, string_dumper) -SaneDumper.add_representer(unicode, string_dumper) -SaneDumper.add_representer(int, partial(string_dumper, _tag=u'tag:yaml.org,2002:int')) -SaneDumper.add_representer(float, partial(string_dumper, _tag=u'tag:yaml.org,2002:float')) - - -def boolean_dumper(dumper, value): - """ - Dump booleans as yes or no. - """ - value = u'yes' if value else u'no' - style = None - return dumper.represent_scalar(u'tag:yaml.org,2002:bool', value, style=style) - -SaneDumper.add_representer(bool, boolean_dumper) diff --git a/tests/test_gen.py b/tests/test_gen.py index 5ece6d76..f7a3423d 100644 --- a/tests/test_gen.py +++ b/tests/test_gen.py @@ -70,12 +70,10 @@ def test_load_inventory(self): else: assert e.message in expected_error_messages - expected = [u'about_resource: .\n' - u'name: AboutCode\n' - u'version: 0.11.0\n' - u'description: |-\n' - u' multi\n' - u' line\n'] + expected = ['about_resource: .\n' + 'name: AboutCode\n' + 'version: 0.11.0\n' + 'description: |\n multi\n line\n'] result = [a.dumps(use_mapping=False, mapping_file=False, with_absent=False, with_empty=False) for a in abouts] assert expected == result @@ -101,15 +99,14 @@ def test_load_inventory_with_mapping(self): else: assert e.message in expected_error_messages - expected = [u'about_resource: .\n' - u'name: AboutCode\n' - u'version: 0.11.0\n' - u'copyright: Copyright (c) nexB, Inc.\n' - u'resource: this.ABOUT\n' - u'description: |-\n' - u' multi\n' - u' line\n' - ] + expected = [ + 'about_resource: .\n' + 'name: AboutCode\n' + 'version: 0.11.0\n' + 'copyright: Copyright (c) nexB, Inc.\n' + 'resource: this.ABOUT\n' + 'description: |\n multi\n line\n' + ] result = [a.dumps(use_mapping, mapping_file=False, with_absent=False, with_empty=False) for a in abouts] assert expected == result @@ -171,9 +168,9 @@ def test_generate(self): expected = (u'about_resource: .\n' u'name: AboutCode\n' u'version: 0.11.0\n' - u'description: |-\n' - u' multi\n' - u' line\n') + u'description: |\n' + u' multi\n' + u' line\n') assert expected == in_mem_result def test_generate_not_overwrite_original_license_file(self): @@ -190,7 +187,7 @@ def test_generate_not_overwrite_original_license_file(self): u'name: AboutCode\n' u'version: 0.11.0\n' u'licenses:\n' - u' - file: this.LICENSE\n') + u' - file: this.LICENSE\n') assert expected == in_mem_result def test_deduplicate(self): diff --git a/tests/test_model.py b/tests/test_model.py index daf15738..a0fc65ef 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -709,24 +709,24 @@ def test_About_dumps(self): a = model.About(test_file) assert [] == a.errors - expected = u'''about_resource: . + expected = '''about_resource: . name: AboutCode version: 0.11.0 copyright: Copyright (c) 2013-2014 nexB Inc. license_expression: apache-2.0 author: - - Jillian Daguil - - Chin Yeung Li - - Philippe Ombredanne - - Thomas Druez -description: |- - AboutCode is a tool - to process ABOUT files. - An ABOUT file is a file. + - Jillian Daguil + - Chin Yeung Li + - Philippe Ombredanne + - Thomas Druez +description: | + AboutCode is a tool + to process ABOUT files. + An ABOUT file is a file. homepage_url: http://dejacode.org licenses: - - file: apache-2.0.LICENSE - key: apache-2.0 + - file: apache-2.0.LICENSE + key: apache-2.0 notice_file: NOTICE owner: nexB Inc. vcs_repository: https://github.com/dejacode/about-code-tool.git diff --git a/thirdparty/saneyaml-0.1-py2.py3-none-any.whl b/thirdparty/saneyaml-0.1-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..c8aa66fac07d8db6362893917fe6b3bffd375358 GIT binary patch literal 5567 zcmaKw1yGc2*T-Q&q#M))1j&UZ1yMqprIubnT9jD2YiSkf&ZUv1TRNq4>5%Sj0SOWL zeBO7y_kAAc^SkajbIpCvb$)Ztoc}r3oTH|Mg-wZpfq{!roRW-G_#CvN*Cp)7x6l}X=UGtgj zQKdMTIBukaf;UIrt{2A;jjGjxywpx+zlCOR-*?2Hg}4nIM5|{i`W{X?9?+i@yX8mt zxS>`>D`tK`<9*I!Y|n_lF1xho-*f%BJjw8|a6Hz|s1WUi$l~gz zRV#E5ru?|gFLSlbE7ZZ|9X;5Loj(I57&8z=aH30=QZt)pInNn(DBc^CXhoZ8Wj7_! z=K(e&mJC{2>KdA#0U9OTGjh74Kl!lRVDjZze%OljID?Z)>L+4&*-y>mEzh0eLeLw1uJ2mS~bA5RYvKzIiC(ND*lpDH)b zK2AwmLZ&AlF>Wc@3mpSF-*=(<3& zh48#IqGgPr@^%?`D@$V(PPH1IZfJ1(t?7%EN*03;*-ZSmx*7o_21d!L4MxP6_leN3Av;*6YfQ84yNb;x~%kWcyCl#G5 zHTPl$)gCWLu)1t=f;L=o;BVYe^kI%dj=ECW1v ziXKoBWOq^!6+v=DM@xG)6iplbye~S0mL1C()GEI>IT#p}L zd#_w95oi&r%ooIc&ejavW4btEt;_T7)ZWn|k&c-Z+!$!FM?W<-7wyy2+FVPw2H8|E zK@d)I6=_AUBj&G*v6L5u{kBxz$~q+VZDIiHW`Wm6TzP8#xQzZXN9k1+p)xejmfZ>< z&3i0a+X4pQxX+i_`HA>Al<#gUX&W{fw`3hgz;V?GUL@m?B(PoW^07!K9t>c)PO%}N zQtgBaLt1VTON?#MNUV`7SSc7@<2D(_RKnpZ@*=@pKXjFCV#;@9%a=Jwp_;dEPs=hL zR_1kf2?5wwI8P}^fB;RNC?S;PwEBIjl){2Gr_gT69*1a=F9c-$lO>MFa(RZ?{7D*{ zKt4O($NP$dI1~(K=KR}Sr=;|ou2+D={6}KfhS@Jj{j9e! zKi<*m(~g~`XI;l%Yy2o6t-#7m>%DY8&dk=}WJlas=kg}Z>+N=HsB7Ixb7SN-eKm;g z0cz+l^*KI1-I9#>J}!ja$D0`h?QUiDI3|x!%;f==B7NenNE8uMt332i$;_B zJk&XLGnVd)l113CSnImDeE;QQaeZ$-L8ee9!g{ zJg!^3m#2%3Rq8c^5fio-Ag~yW$jR8X|E{mbhxY)hlxh!lFUIVy?(<#Bhl7HqY5pvf zDyX42){y?C^?f+oRi>3O=wtC(=;Lz=A$TjHpvL<`Snvu|nBoy6CZ>-EowVTSey-#2 ztZ>;c>n!^IfMv^?7xmi_MK4$Is;vVk;NApjYy5L%L#&kuZ+7f%ssWhsIt%wg{I`;yq zc)W2v%9!M+dYthOjFGuPj)bb0Zp;0b{^@HYYD<2;CkJYS%VfPdfa8~{+j(r#JG}dV z02Q8P^%%FSt5$M^0f_GvQN2-SJDn(@xk4{?1mo+=2wl0GB)w!GZQ<;=UjHv6uKSdx z-QF;b0g48?UFs;}x~Ug~>1RIcpQSRAR{>lo=X}wF?gZT@NR&Rx@SW<9D92NbG4{bt zny`^xbP~uwzp`*zZf1o`QB7C|<@9!X^-aWwy`sIwQ2y!|m3|L`1$hI;V+#vSUy~w% zHYruUJYbrCn^L5xJPnXy>y)q(SRxlD0uk@-14%%dZHL00W1yD4fi&mM=@|~0n_XVc z2uDZ62lJazNNI#{f%D}ZSl>(wV1{8!J5&Gqn|9{lqiCB(GTrw)L3lq*EujbYV(j=GrFY?rhECQ=Je2`^Y+X_BzjqUHpz=x!=4>3PIl&LwY-htj@wTP zq*O1hligVjX5ayg*UfE)bj}FyLt}(qzk3Vb#iz4?xKs1!pTJ7y@C_rI1XP=+A5aK? z{Nj@G0aL8@gEt`yY-S2bYWF~hrlhBP@t3S7{{#*1NT$Os@5A;QMZ2@ZiPHsbxzlaA z9)h7KraAE?i;}$O&IeW#xnY4eFEj|$=@cW;gyHggNXI1gQNWv4Z{Lga?Ho$)k%jws za~(i%@gX+F!MCId?WJgz{-jhTAHhgs3-5QjRlrAuBb~<~p)PghNP}(IXhSH}AU4t1 zN=lw$F`#dmZ}x3eV5+s=Rz-~#XZGM~71X4!E1c>JU_13=2_)Dhp@TzsL3ArhukRe_ zC-xW=*hpJT1UZnMj3llogzT_4NV{+>#1k@u--NzL=%|KGq=?mNSXy*-;7h+P>bo-> zUv|z;aKq=09-!EJxXT7kxeWYDuR2>kFh8TZyD(?wSBpeK)z8Etu6^Zt#-FT>j*{_{ z%MM0qeaqv;aa_#B*~?>9njKC)^X&sjfLFLW8N32XQT72;ZPEM`WP;GV^@si@YBPVnJS_K9=kAvU7fh6qQu}|wHCKI*#|$3 zAISI7#3Gvbp5jN5^opEvE5AS|OxXG$DBr+gn&`=QfeI%1Hm(gReOsCW8cD$1Nt4Zc z6~~7*-{-rP9jk4~jnna4{fWzXN{hB3Fas^u84q=ep*z_ew$SGY2==+pB1wUw6k5Zj41v1 z(bLT8HvYaUPn^PMSOzOeRXJ|hXk#oJdA|@3*>TfIh#H*G9Pq6*>(KXk!xp(-GsR+x z&i9)SY!8fB%efDZz+749ho&D4nT`htx;5fLysD45(dr5WtH;8RtnuKr%9`Vb-F2^Q zO9&|X)H6&=(%kEn?M(G^qhSPu3cXNYU;Nx0-34dm_T$Pt5gaEn3VOmE*DAfx1hx@| z*?1*Em+Qp(t#HmyCye7MU1Mb#m8b<$)?#=@dd&6B$`k-`Zy;kyB>OgMypy?&df7m? zLzrMxud}={KK^A9N%ZGQi7rRO*|;IFg3oweD2aTaRRHh5^O>F&4;~~s#&NA~-8cgAp+Y1xmNw2(=3i@&qL5B0aJ3~0lfLCiVBMs7 z0A9i*)KincN9F-_l+A+@`TS6U+>sNh^n@jNT7kInf{bVygY8SA#n_DXW?S zmA+&CYSOSVFzEl*t8wvi^KqM4I5~4!*qYh#D9gemWh7ye@v+Kw;}WDFj;_I*xElL{ zcz`~D44fFIJRVe)`LO4F+OUG&%*|%acS(n>0CzPe_F^p!X})Q&q}fW2#VX$GVWEr@ z>dak}&yE%O0WzW3v}ka;&@v8ONNQ^s=X$n!tWPFdcKFmU7u&RAeI#U}_xh{-sPM@; z!osq=l=)@< zx^Dt!ebrmgW;i<}Sx&JCgWEqI5wE)2tHa4g07Huim}H*9q)ZK%^Gz0n2nnM@?6fq; zUJu4vNWm+Mz-XI$_<9O1**RPB>|Sr=V6RwUY-v?OVxpZc>|{HKoniLhyQLUd8(GdqXu5luvs;9PMe$-_O`jC-7i9Atn&D- zO0R0r=y^5?io_!+6D9aK$)44>>IX1;^U018_fMlMN>q$w%)O-@H?}JyrKz#N>D#lB zC9fY<(5-=)txM!*s#OcmTJOtxDyb%nf68Xwj_F?XFWLA4Ffi!;EE^S7n1ZydsYaV^ zDlcjCH8}k|PF_#msc@E}aR!u62IfkE2m$%a6Iyd?@K%mQSY;GpM#V^v^ozx?3)iGH zTOLg9U106gCEwPOIpTVCJXC>kM?k7-lnWhoW+0R5Vf?O)zSMo6T=viT(=dH8h9kc4I7~;fDOmmq!Z=}XM`4;Jkrr)UNyCORmAWB{NTC$gcn2-2UD!5 z$0C`$J_uTDHGW99KHlTy3+nH9O3jfDl@lACc#e*w~|s8LZVC;sHpS^X-z zzi%f~xiO$tISIw2ZbhG3+I?&)N}oKvWsX52548S7?or>y4L)@Qr=Om7eBBA#+IQ{` zaqj z<#OpvmEZ?pp$-Ud)LwOD(dpNxK`Ix*z2%*^GYXyJ|BNLxHx6O+D~7hejP}p5)Rvc( zg%S+QD-VtmDRFTOp@%CqcqjOm9aRUFxD+}2cs^;!EB0}62lL!5P#@tP;TIg^pIXQ7 zALbeT#yLd<;!+&!gMQKgv2!SI;446DH9+|SV}k?3N|VayqW8h~nPlpI6WN3*ZRub0 zBfMY6{O2EYwzD^~Hgz?%=5}^>2CJZzN}%Y$6~J$>pqcRaN52%?`pfe<1zL^<;7Ya2M5Q{fjXdDvlOJ^7*qf7$O{1^d8moe6>XNO9IaJAS z#GfGHV{hClre4-|oNA|LVkDy7z8&Wo&tc1bc*C&8{4QB2mAMdXBY8_^NXd-(y5C)Y zYcRze@*NU+#zxD}@{)*6o+Jz2AQ>zdQ$=UNDWGA;ZWb-Ky`g@zUUGSN;K-?zgt3R$ zXkqA_L{^a)`TV8w)Wu7p-FaeL2mV&f5|47Et~3NfX5a59=((7@`O}oR^iEq2?5iVf zHc71CV^>cM$&71P*NIZE1h4}syTB&1LMp$GlPc~m8QFAix@-0e=_gtX9ba{!Q*R7| zUsIQzc+`|I0hCyO9{K%>*1uMj|6kkR27rHe z{J-v>f4gB|RQVVGnjrqK;~#FL-(kO3rT@a#Nd5@>ca8cT^?OqM7ga{~N7TPk&R@rIem!-+GT|%5UvK{d`}Z3^ literal 0 HcmV?d00001 diff --git a/thirdparty/saneyaml-0.1-py2.py3-none-any.whl.ABOUT b/thirdparty/saneyaml-0.1-py2.py3-none-any.whl.ABOUT new file mode 100644 index 00000000..ae857b3e --- /dev/null +++ b/thirdparty/saneyaml-0.1-py2.py3-none-any.whl.ABOUT @@ -0,0 +1,20 @@ +about_resource: saneyaml-0.1-py2.py3-none-any.whl +attribute: true +checksum_md5: 53509e4f1ee9f6565158163a56513b7c +checksum_sha1: a66920309794ead47711473a21f1c7d003acd005 +contact: http://www.nexb.com/contactus.html +copyright: Copyright (c) nexB Inc. and others. +download_url: https://files.pythonhosted.org/packages/21/13/3d639adcd97fc22cfc4acfd23330cc26f96a85a5b992d93086684e24ac14/saneyaml-0.1-py2.py3-none-any.whl +homepage_url: https://github.com/nexB/saneyaml +license_expression: apache-2.0 +licenses: +- file: apache-2.0.LICENSE + key: apache-2.0 + name: Apache License 2.0 +name: saneyaml +owner: nexB +owner_url: http://www.nexb.com/ +package_url: pkg:pypi/saneyaml@0.1 +track_changes: true +vcs_repository: https://github.com/nexB/saneyaml +version: '0.1' From 82fce52fbeb3580000df06d832373974c667d77f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 3 Jan 2019 20:11:36 +0100 Subject: [PATCH 3/4] Bump version to 3.3.1 and update CHANGELOG Signed-off-by: Philippe Ombredanne --- docs/CHANGELOG.rst | 11 +++++++++++ setup.py | 2 +- src/attributecode/__init__.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.rst b/docs/CHANGELOG.rst index 8a987e6c..6a487e56 100644 --- a/docs/CHANGELOG.rst +++ b/docs/CHANGELOG.rst @@ -1,3 +1,14 @@ +2018-01-03 + + Release 3.3.1 + + * Add new Jinja2 custom template filters to multi_sort to sort and + unique_together to compute unique lists. Both filter take an attributes + list of attribute names and use all these attribute names to sort or + compute unique values. + * Use saneyaml library to dump and load YAML + + 2018-11-15 Release 3.3.0 diff --git a/setup.py b/setup.py index 78dca192..55c1fcfb 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def read(*names, **kwargs): setup( name='aboutcode-toolkit', - version='3.3.0', + version='3.3.1', license='Apache-2.0', description=( 'AboutCode-toolkit is a tool to document the provenance (origin and license) of ' diff --git a/src/attributecode/__init__.py b/src/attributecode/__init__.py index 0397bfa8..56941c3b 100644 --- a/src/attributecode/__init__.py +++ b/src/attributecode/__init__.py @@ -27,7 +27,7 @@ basestring = str # Python 3 #NOQA -__version__ = '3.3.0' +__version__ = '3.3.1' __about_spec_version__ = '3.1' From 469a061e166d4be0b5a316e66983731eaad9a35f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 3 Jan 2019 20:22:29 +0100 Subject: [PATCH 4/4] Use OrderedDict when dumping ABOUT files This ensure that we have a stable order on all OSes Signed-off-by: Philippe Ombredanne --- src/attributecode/model.py | 4 ++-- tests/test_model.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/attributecode/model.py b/src/attributecode/model.py index 016bafb7..ad81f1ff 100644 --- a/src/attributecode/model.py +++ b/src/attributecode/model.py @@ -1081,7 +1081,7 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e If with_absent, include absent (not present) fields. If with_empty, include empty fields. """ - about_data = {} + about_data = OrderedDict() # Group the same license information (name, url, file) together license_key = [] license_name = [] @@ -1110,7 +1110,7 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e # Group the same license information in a list license_group = list(zip_longest(license_key, license_name, license_file, license_url)) for lic_group in license_group: - lic_dict = {} + lic_dict = OrderedDict() if lic_group[0]: lic_dict['key'] = lic_group[0] if lic_group[1]: diff --git a/tests/test_model.py b/tests/test_model.py index a0fc65ef..c46f7ed5 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -725,8 +725,8 @@ def test_About_dumps(self): An ABOUT file is a file. homepage_url: http://dejacode.org licenses: - - file: apache-2.0.LICENSE - key: apache-2.0 + - key: apache-2.0 + file: apache-2.0.LICENSE notice_file: NOTICE owner: nexB Inc. vcs_repository: https://github.com/dejacode/about-code-tool.git