diff --git a/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md b/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md index 688105c4..33830a3f 100644 --- a/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md +++ b/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md @@ -287,6 +287,19 @@ and "version" columns and no other column: - name - version +* exclude_fields: +An optional list of field names that should be excluded in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be excluded +in the target CSV/JSON must be listed. Excluding standard or required fields will cause +an error. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. + +For instance with this configuration the target CSV/JSON will not contain the "type" +and "temp" fields: + exclude_fields: + - type + - temp + ## Run gen to Generate AboutCode Toolkit Files diff --git a/src/attributecode/transform.py b/src/attributecode/transform.py index c14e4add..9caf7617 100644 --- a/src/attributecode/transform.py +++ b/src/attributecode/transform.py @@ -109,6 +109,10 @@ def transform_csv(rows, transformer): data = list(transformer.filter_fields(data)) field_names = [c for c in field_names if c in transformer.field_filters] + if transformer.exclude_fields: + data = list(transformer.filter_excluded(data)) + field_names = [c for c in field_names if c not in transformer.exclude_fields] + errors = transformer.check_required_fields(data) return field_names, data, errors @@ -162,6 +166,11 @@ def process_json_keys(data, renamings, transformer): new_data = list(transformer.filter_fields(new_data)) else: new_data = list(new_data) + + if transformer.exclude_fields: + new_data = list(transformer.filter_excluded(new_data)) + else: + new_data = list(new_data) errors = transformer.check_required_fields(new_data) return new_data, errors @@ -212,6 +221,19 @@ def process_json_keys(data, renamings, transformer): field_filters: - name - version + +* exclude_fields: +An optional list of field names that should be excluded in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be excluded +in the target CSV/JSON must be listed. Excluding standard or required fields will cause +an error. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. + +For instance with this configuration the target CSV/JSON will not contain the "type" +and "temp" fields: + exclude_fields: + - type + - temp ''' @@ -222,6 +244,7 @@ class Transformer(object): field_renamings = attr.attrib(default=attr.Factory(dict)) required_fields = attr.attrib(default=attr.Factory(list)) field_filters = attr.attrib(default=attr.Factory(list)) + exclude_fields = attr.attrib(default=attr.Factory(list)) # a list of all the standard fields from AboutCode toolkit standard_fields = attr.attrib(default=attr.Factory(list), init=False) @@ -245,6 +268,7 @@ def default(cls): field_renamings={}, required_fields=[], field_filters=[], + exclude_fields=[], ) @classmethod @@ -259,6 +283,7 @@ def from_file(cls, location): field_renamings=data.get('field_renamings', {}), required_fields=data.get('required_fields', []), field_filters=data.get('field_filters', []), + exclude_fields=data.get('exclude_fields', []), ) def check_required_fields(self, data): @@ -317,6 +342,17 @@ def filter_fields(self, data): items = ((k, v) for k, v in entry.items() if k in field_filters) yield OrderedDict(items) + def filter_excluded(self, data): + """ + Yield transformed dicts from a `data` list of dicts excluding + fields with names in the `exclude_fields`of this Transformer. + Return the data unchanged if no `exclude_fields` exists. + """ + exclude_fields = set(self.clean_fields(self.exclude_fields)) + for entry in data: + items = ((k, v) for k, v in entry.items() if k not in exclude_fields) + yield OrderedDict(items) + def check_duplicate_fields(field_names): """ diff --git a/tests/testdata/test_cmd/help/about_transform_config_help.txt b/tests/testdata/test_cmd/help/about_transform_config_help.txt index d7bb5066..44f5fd89 100644 --- a/tests/testdata/test_cmd/help/about_transform_config_help.txt +++ b/tests/testdata/test_cmd/help/about_transform_config_help.txt @@ -44,3 +44,16 @@ and "version" fields and no other field: - name - version +* exclude_fields: +An optional list of field names that should be excluded in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be excluded +in the target CSV/JSON must be listed. Excluding standard or required fields will cause +an error. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. + +For instance with this configuration the target CSV/JSON will not contain the "type" +and "temp" fields: + exclude_fields: + - type + - temp + diff --git a/tests/testdata/test_transform/configuration b/tests/testdata/test_transform/configuration index 77ebb91a..89d1e8b3 100644 --- a/tests/testdata/test_transform/configuration +++ b/tests/testdata/test_transform/configuration @@ -5,6 +5,9 @@ field_filters: - about_resource - name - version + - temp required_fields: - name - - version \ No newline at end of file + - version +exclude_fields: + - temp \ No newline at end of file diff --git a/tests/testdata/test_transform/configuration_scancode b/tests/testdata/test_transform/configuration_scancode index b5529962..ecc5095b 100644 --- a/tests/testdata/test_transform/configuration_scancode +++ b/tests/testdata/test_transform/configuration_scancode @@ -5,7 +5,9 @@ field_filters: - name - new_extension - about_resource + - type required_fields: - name - +exclude_fields: + - type diff --git a/tests/testdata/test_transform/input.csv b/tests/testdata/test_transform/input.csv index 7f863d86..a9f18446 100644 --- a/tests/testdata/test_transform/input.csv +++ b/tests/testdata/test_transform/input.csv @@ -1,2 +1,2 @@ -Directory/Filename,Component,version,notes -/tmp/test.c, test.c,1,test +Directory/Filename,Component,version,notes,temp +/tmp/test.c, test.c,1,test,foo diff --git a/tests/testdata/test_transform/input.json b/tests/testdata/test_transform/input.json index 73981241..f088ea2a 100644 --- a/tests/testdata/test_transform/input.json +++ b/tests/testdata/test_transform/input.json @@ -2,5 +2,6 @@ "Directory/Filename": "/aboutcode-toolkit/", "Component": "AboutCode-toolkit", "version": "1.2.3", - "note": "test" + "note": "test", + "temp": "foo" } \ No newline at end of file diff --git a/tests/testdata/test_transform/input_as_array.json b/tests/testdata/test_transform/input_as_array.json index f6c07108..0bfa970c 100644 --- a/tests/testdata/test_transform/input_as_array.json +++ b/tests/testdata/test_transform/input_as_array.json @@ -2,11 +2,13 @@ { "Directory/Filename": "/aboutcode-toolkit/", "Component": "AboutCode-toolkit", - "version": "1.0" + "version": "1.0", + "temp": "fpp" }, { "Directory/Filename": "/aboutcode-toolkit1/", "Component": "AboutCode-toolkit1", - "version": "1.1" + "version": "1.1", + "temp": "foo" } ] \ No newline at end of file