Do not fail if CSV contains non-UTF mojibake

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/attributecode/transform.py

@@ -258,7 +258,8 @@ def read_csv_rows(location):
     """
     Yield rows (as a list of values) from a CSV file at `location`.
     """
-    with io.open(location, encoding='utf-8') as csvfile:
+    # note: Excel can produce unreadable UTF files
+    with io.open(location, encoding='utf-8', errors='replace') as csvfile:
         reader = csv.reader(csvfile)
         for row in reader:
             yield row

tests/test_transform.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# -*- coding: utf8 -*-
+
+# ============================================================================
+#  Copyright (c) 2014-2017 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+# ============================================================================
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import unittest
+
+from attributecode import transform
+
+from testing_utils import get_test_loc
+
+
+class TransformTest(unittest.TestCase):
+
+    def test_read_csv_rows_can_read_invalid_utf8(self):
+        test_file = get_test_loc('test_transform/mojibake.csv')
+        list(transform.read_csv_rows(test_file))

tests/testdata/test_transform/mojibake.csv

@@ -0,0 +1 @@
+ Action Item Ref# , Inventory Ref# ,Product / Subsystem,Resource Path,Resource Name,Owner,Component,Version,License Category,License Name(s),License Expression,Copyright,Attribution,Redistribution,Modified,Internal Use Only,Auditor Notes,Code Download URL,License Reference,Detected License Expression,Resource Type,Resource Size,Resource SHA1,Resource File Type,Language,about_file,notice_text_file,90160,Mobile / iosapp,/iosapp/Pods/NUIParse,NUIParse,,NUIParse,v 1.3,Permissive,BSD-3-Clause,bsd-new,"Copyright (c) 2011, Thomas Cook",x,,,,"Per the .markdown file, I've used ""Copyright (c) 2011, Thomas Davie"" as the top-level copyright.  LICENSE has a Thomas Davie copyright, while a number of the other files have this: Copyright 2011 In The Beginning... All rights reserved.  Some have a 2012 date for the ""In The Beginningƒ¿½"" copyright.  2 files have ""Copyright (c) 2014 Ignition Soft.""  Version found in /iosapp/Podfile.lock",,,,directory,0,,,,/about_files/iosapp/Pods/NUIParse/,