Handle CSV data de/encoding

Author: fungjj92

opentreemap/exporter/tests.py

@@ -38,16 +38,19 @@ def setUp(self):
                                                           password='bar')
 
     def assertCSVRowValue(self, csv_file, row_index, headers_and_values):
-        csvreader = csv.reader(csv_file, delimiter=b",")
-        rows = list(csvreader)
+        # decode bytes object into string list required by the csv reader
+        str_rows = [line.decode('utf-8') for line in csv_file]
 
         # strip the BOM out
-        rows[0][0] = rows[0][0][3:]
+        str_rows[0] = str_rows[0][1:]
 
-        self.assertTrue(len(rows) > 1)
+        csvreader = csv.reader(str_rows, delimiter=",")
+        reader_rows = list(csvreader)
+
+        self.assertTrue(len(reader_rows) > 1)
         for (header, value) in headers_and_values.items():
-            target_column = rows[0].index(header)
-            self.assertEqual(value, rows[row_index][target_column])
+            target_column = reader_rows[0].index(header)
+            self.assertEqual(value, reader_rows[row_index][target_column])
 
     def assertTaskProducesCSV(self, user, model, assert_fields_and_values):
         self._assertTaskProducesCSVBase(user, model, assert_fields_and_values)
@@ -82,6 +85,7 @@ def assertPsuedoAsyncTaskWorks(self, model,
 
         job_id = ctx['job_id']
         job = ExportJob.objects.get(pk=job_id)
+
         self.assertCSVRowValue(job.outfile, 1,
                                {assertion_field: assertion_value})
 
@@ -213,16 +217,20 @@ class UserExportsTest(UserExportsTestCase):
 
     def get_csv_data_with_base_assertions(self):
         resp = users_csv(make_request(), self.instance)
-        reader = csv.reader(resp)
 
-        # Skip BOM and entry line
-        next(reader)
-        next(reader)
+        # decode bytes object into string list required by the csv reader
+        str_rows = [line.decode('utf-8') for line in resp]
 
+        # strip the BOM and entry line out
+        reader = csv.reader(str_rows[2:])
+
+        # grab and strip the header
         header = next(reader)
 
-        data = [dict(list(zip(header, [x.decode('utf8') for x in row])))
-                for row in reader]
+        reader_rows = list(reader)
+
+        data = (lambda h=header, rows=reader_rows:
+                [dict(list(zip(h, [x for x in row]))) for row in rows])()
 
         commander, user1data, user2data = data
         self.assertEqual(commander['username'], self.commander.username)

opentreemap/importer/util.py

@@ -34,7 +34,7 @@ def _as_utf8(f):
 
 
 def _guess_dialect_and_reset_read_pointer(f):
-    dialect = csv.Sniffer().sniff(_as_utf8(f).read(4096), delimiters=',\t')
+    dialect = csv.Sniffer().sniff(_as_utf8(f).read(4096).decode(), delimiters=',\t')
     f.seek(0)
     return dialect
 

opentreemap/importer/views.py

@@ -707,7 +707,7 @@ def process_csv(request, instance, import_type, **kwargs):
     filename = list(files.keys())[0]
     file_obj = files[filename]
 
-    file_obj = io.BytesIO(decode(file_obj.read()).encode('utf-8'))
+    file_obj = io.BytesIO((file_obj.read().encode('utf-8')))
 
     owner = request.user
     ImportEventModel = get_import_event_model(import_type)