diff --git a/opentreemap/api/auth.py b/opentreemap/api/auth.py index b81cea029..604b54587 100644 --- a/opentreemap/api/auth.py +++ b/opentreemap/api/auth.py @@ -9,7 +9,7 @@ import urllib.parse import urllib.error -from django.http import HttpResponse +from django.http import HttpResponse, RawPostDataException from django.contrib.auth import authenticate @@ -37,19 +37,32 @@ def get_signature_for_request(request, secret_key): sign_string = '\n'.join([httpverb, hostheader, request_uri, paramstr]) - # Sometimes reeading from body fails, so try reading as a file-like + # Sometimes reading from body fails, so try reading as a file-like object try: - body_encoded = base64.b64encode(request.body) - except: - body_encoded = base64.b64encode(request.read()) + body_decoded = base64.b64encode(request.body).decode() + except RawPostDataException: + body_decoded = base64.b64encode(request.read()).decode() - if body_encoded: - sign_string += body_encoded + if body_decoded: + sign_string += body_decoded + + try: + binary_secret_key = secret_key.encode() + except (AttributeError, UnicodeEncodeError): + binary_secret_key = secret_key sig = base64.b64encode( - hmac.new(secret_key, sign_string, hashlib.sha256).digest()) + hmac.new( + binary_secret_key, + sign_string.encode(), + hashlib.sha256 + ).digest() + ) + + if sig is None: + return sig - return sig + return sig.decode() def create_401unauthorized(body="Unauthorized"): @@ -67,11 +80,15 @@ def firstmatch(regx, strg): return m.group(1) -def decodebasicauth(strg): - if strg is None: +def split_basicauth(strg): + """ + Returns username, password from decoded, + stringified, basic auth credentials + """ + if strg is None or len(strg) == 0: return None else: - m = re.match(r'([^:]*)\:(.*)', base64.decodestring(strg)) + m = re.match(r'([^:]*)\:(.*)', strg) if m is not None: return (m.group(1), m.group(2)) else: @@ -79,7 +96,22 @@ def decodebasicauth(strg): def parse_basicauth(authstr): - auth = decodebasicauth(firstmatch('Basic (.*)', authstr)) + string_wrapped_binary_credentials = firstmatch("Basic (.*)", authstr) + if string_wrapped_binary_credentials is None: + return None + + # tease bytes-like object out of string, i.e. "b'credentials'" + reg_exp = r"'(.*?)\'" + parsed_credentials = re.search(r"'(.*?)\'", string_wrapped_binary_credentials) + str_credentials = parsed_credentials.groups()[0] + + # decode the binary encoded credentials + decoded_str_credentials = base64.decodebytes( + bytes(str_credentials, 'utf-8') + ).decode() + + auth = split_basicauth(decoded_str_credentials) + if auth is None: return None else: diff --git a/opentreemap/api/models.py b/opentreemap/api/models.py index 191bd0cf0..4e48d163c 100755 --- a/opentreemap/api/models.py +++ b/opentreemap/api/models.py @@ -31,7 +31,7 @@ def __unicode__(self): def create(clz, user=None): secret_key = base64.urlsafe_b64encode(os.urandom(64)) access_key = base64.urlsafe_b64encode(uuid.uuid4().bytes)\ - .replace('=', '') + .replace(b'=', b'') return APIAccessCredential.objects.create( user=user, access_key=access_key, secret_key=secret_key) diff --git a/opentreemap/api/plots.py b/opentreemap/api/plots.py index 5151d6f36..a50041877 100644 --- a/opentreemap/api/plots.py +++ b/opentreemap/api/plots.py @@ -8,6 +8,7 @@ from django.shortcuts import get_object_or_404 from django.contrib.gis.geos import Point from django.contrib.gis.measure import D +from django.http import RawPostDataException from django_tinsel.exceptions import HttpBadRequestException @@ -50,8 +51,7 @@ def plots_closest_to_point(request, instance, lat, lng): distance = float(request.GET.get( 'distance', settings.MAP_CLICK_RADIUS)) except ValueError: - raise HttpBadRequestException( - 'The distance parameter must be a number') + raise HttpBadRequestException('The distance parameter must be a number') plots = Plot.objects.distance(point)\ .filter(instance=instance)\ @@ -72,7 +72,16 @@ def update_or_create_plot(request, instance, plot_id=None): # The API communicates via nested dictionaries but # our internal functions prefer dotted pairs (which # is what inline edit form users) - request_dict = json.loads(request.body) + # Sometimes reading from body fails, so try reading as a file-like object + try: + body_decoded = request.body.decode() + except RawPostDataException: + body_decoded = request.read().decode() + + if body_decoded: + request_dict = json.loads(body_decoded) + else: + request_dict = {} data = {} diff --git a/opentreemap/api/tests.py b/opentreemap/api/tests.py index 9e7b268df..e47b21c51 100644 --- a/opentreemap/api/tests.py +++ b/opentreemap/api/tests.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from io import StringIO +from io import BytesIO from json import loads, dumps from urllib.parse import urlparse @@ -14,6 +14,7 @@ import datetime import psycopg2 from unittest.case import skip +from django_tinsel.exceptions import HttpBadRequestException from django.db import connection from django.contrib.auth.models import AnonymousUser @@ -96,11 +97,11 @@ def send_json_body(url, body_object, client, method, user=None): are posting form data, so you need to manually setup the parameters to override that default functionality. """ - body_string = dumps(body_object) - body_stream = StringIO(body_string) + body_binary_string = dumps(body_object).encode() + body_stream = BytesIO(body_binary_string) parsed_url = urlparse(url) client_params = { - 'CONTENT_LENGTH': len(body_string), + 'CONTENT_LENGTH': len(body_binary_string), 'CONTENT_TYPE': 'application/json', 'PATH_INFO': _get_path(parsed_url), 'QUERY_STRING': parsed_url[4], @@ -375,8 +376,8 @@ def test_locations_plots_endpoint_max_plots_param_must_be_a_number(self): API_PFX, self.instance.url_name)) self.assertEqual(response.status_code, 400) self.assertEqual(response.content, - 'The max_plots parameter must be ' - 'a number between 1 and 500') + b'The max_plots parameter must be ' + b'a number between 1 and 500') def test_locations_plots_max_plots_param_cannot_be_greater_than_500(self): response = get_signed( @@ -385,8 +386,8 @@ def test_locations_plots_max_plots_param_cannot_be_greater_than_500(self): API_PFX, self.instance.url_name)) self.assertEqual(response.status_code, 400) self.assertEqual(response.content, - 'The max_plots parameter must be ' - 'a number between 1 and 500') + b'The max_plots parameter must be ' + b'a number between 1 and 500') response = get_signed( self.client, "%s/instance/%s/locations/0,0/plots?max_plots=500" % @@ -402,8 +403,8 @@ def test_locations_plots_endpoint_max_plots_param_cannot_be_less_than_1( self.assertEqual(response.status_code, 400) self.assertEqual(response.content, - 'The max_plots parameter must be a ' - 'number between 1 and 500') + b'The max_plots parameter must be a ' + b'number between 1 and 500') response = get_signed( self.client, "%s/instance/%s/locations/0,0/plots?max_plots=1" % @@ -419,7 +420,7 @@ def test_locations_plots_endpoint_distance_param_must_be_a_number(self): self.assertEqual(response.status_code, 400) self.assertEqual(response.content, - 'The distance parameter must be a number') + b'The distance parameter must be a number') response = get_signed( self.client, @@ -472,7 +473,7 @@ def test_create_plot_with_tree(self): data, self.client, self.user) self.assertEqual(200, response.status_code, - "Create failed:" + response.content) + "Create failed:" + response.content.decode()) # Assert that a plot was added self.assertEqual(plot_count + 1, Plot.objects.count()) @@ -509,7 +510,7 @@ def test_create_plot_with_invalid_tree_returns_400(self): self.assertEqual(400, response.status_code, "Expected creating a million foot " - "tall tree to return 400:" + response.content) + "tall tree to return 400:" + response.content.decode()) body_dict = loads(response.content) self.assertTrue('fieldErrors' in body_dict, @@ -548,7 +549,7 @@ def test_create_plot_with_geometry(self): data, self.client, self.user) self.assertEqual(200, response.status_code, - "Create failed:" + response.content) + "Create failed:" + response.content.decode()) # Assert that a plot was added self.assertEqual(plot_count + 1, Plot.objects.count()) @@ -1334,8 +1335,8 @@ def setUp(self): 'sort_key': 'Date'} ] self.instance.save() - self.instance.logo.save(Instance.test_png_name, - File(open(Instance.test_png_path, 'r'))) + with open(Instance.test_png_path, 'rb') as f: + self.instance.logo.save(Instance.test_png_name, f) def test_returns_config_colors(self): request = sign_request_as_user(make_request(), self.user) @@ -1568,7 +1569,7 @@ def _test_post_photo(self, path): self.instance.url_name, plot_id) - with open(path) as img: + with open(path, 'rb') as img: req = self.factory.post( url, {'name': 'afile', 'file': img}) @@ -1621,7 +1622,7 @@ def testUploadPhoto(self): url = reverse('update_user_photo', kwargs={'version': 3, 'user_id': peon.pk}) - with open(TreePhotoTest.test_jpeg_path) as img: + with open(TreePhotoTest.test_jpeg_path, 'rb') as img: req = self.factory.post( url, {'name': 'afile', 'file': img}) @@ -1648,7 +1649,7 @@ def testCanOnlyUploadAsSelf(self): grunt = make_user(username='grunt', password='pw') grunt.save() - with open(TreePhotoTest.test_jpeg_path) as img: + with open(TreePhotoTest.test_jpeg_path, 'rb') as img: req = self.factory.post( url, {'name': 'afile', 'file': img}) @@ -1883,7 +1884,7 @@ def testTimestampVoidsSignature(self): acred = APIAccessCredential.create() url = ('http://testserver.com/test/blah?' 'timestamp=%%s&' - 'k1=4&k2=a&access_key=%s' % acred.access_key) + 'k1=4&k2=a&access_key=%s' % acred.access_key.decode()) curtime = datetime.datetime.now() invalid = curtime - datetime.timedelta(minutes=100) @@ -1902,7 +1903,7 @@ def testPOSTBodyChangesSig(self): url = "%s/i/plots/1/tree/photo" % API_PFX def get_sig(path): - with open(path) as img: + with open(path, 'rb') as img: req = self.factory.post( url, {'name': 'afile', 'file': img}) @@ -1949,14 +1950,14 @@ def testMalformedTimestamp(self): url = ('http://testserver.com/test/blah?' 'timestamp=%%s&' - 'k1=4&k2=a&access_key=%s' % acred.access_key) + 'k1=4&k2=a&access_key=%s' % acred.access_key.decode()) req = self.sign_and_send(url % ('%sFAIL' % timestamp), acred.secret_key) self.assertEqual(req.status_code, 400) - req = self.sign_and_send(url % timestamp, acred.secret_key) + req = self.sign_and_send(url % timestamp, acred.secret_key.decode()) self.assertRequestWasSuccess(req) @@ -1972,7 +1973,7 @@ def testMissingAccessKey(self): self.assertEqual(req.status_code, 400) - req = self.sign_and_send('%s&access_key=%s' % (url, acred.access_key), + req = self.sign_and_send('%s&access_key=%s' % (url, acred.access_key.decode()), acred.secret_key) self.assertRequestWasSuccess(req) @@ -1987,9 +1988,8 @@ def testAuthenticatesAsUser(self): req = self.sign_and_send('http://testserver.com/test/blah?' 'timestamp=%s&' 'k1=4&k2=a&access_key=%s' % - (timestamp, acred.access_key), - acred.secret_key) - + (timestamp, acred.access_key.decode()), + acred.secret_key.decode()) self.assertEqual(req.user.pk, peon.pk) @@ -2003,7 +2003,7 @@ def test_401(self): self.assertEqual(ret.status_code, 401) def test_ok(self): - auth = base64.b64encode("jim:password") + auth = base64.b64encode(b"jim:password") withauth = {"HTTP_AUTHORIZATION": "Basic %s" % auth} ret = get_signed(self.client, "%s/user" % API_PFX, **withauth) @@ -2015,14 +2015,14 @@ def test_malformed_auth(self): ret = get_signed(self.client, "%s/user" % API_PFX, **withauth) self.assertEqual(ret.status_code, 401) - auth = base64.b64encode("foobar") + auth = base64.b64encode(b"foobar") withauth = {"HTTP_AUTHORIZATION": "Basic %s" % auth} ret = get_signed(self.client, "%s/user" % API_PFX, **withauth) self.assertEqual(ret.status_code, 401) def test_bad_cred(self): - auth = base64.b64encode("jim:passwordz") + auth = base64.b64encode(b"jim:passwordz") withauth = {"HTTP_AUTHORIZATION": "Basic %s" % auth} ret = get_signed(self.client, "%s/user" % API_PFX, **withauth) @@ -2034,7 +2034,7 @@ def test_user_has_rep(self): ijim.reputation = 1001 ijim.save() - auth = base64.b64encode("jim:password") + auth = base64.b64encode(b"jim:password") withauth = dict(list(self.sign.items()) + [("HTTP_AUTHORIZATION", "Basic %s" % auth)]) diff --git a/opentreemap/api/user.py b/opentreemap/api/user.py index 638bcf781..e9c74d93a 100644 --- a/opentreemap/api/user.py +++ b/opentreemap/api/user.py @@ -138,7 +138,7 @@ def wrapper(request, *args, **kwargs): # You can't directly set a new request body # (http://stackoverflow.com/a/22745559) request._body = body - request._stream = BytesIO(body) + request._stream = BytesIO(body.encode()) return user_view_fn(request, *args, **kwargs) diff --git a/opentreemap/exporter/tests.py b/opentreemap/exporter/tests.py index 396f691f3..ee72da3d7 100644 --- a/opentreemap/exporter/tests.py +++ b/opentreemap/exporter/tests.py @@ -38,16 +38,19 @@ def setUp(self): password='bar') def assertCSVRowValue(self, csv_file, row_index, headers_and_values): - csvreader = csv.reader(csv_file, delimiter=b",") - rows = list(csvreader) + # decode bytes object into string list required by the csv reader + str_rows = [line.decode('utf-8') for line in csv_file] # strip the BOM out - rows[0][0] = rows[0][0][3:] + str_rows[0] = str_rows[0][1:] - self.assertTrue(len(rows) > 1) + csvreader = csv.reader(str_rows, delimiter=",") + reader_rows = list(csvreader) + + self.assertTrue(len(reader_rows) > 1) for (header, value) in headers_and_values.items(): - target_column = rows[0].index(header) - self.assertEqual(value, rows[row_index][target_column]) + target_column = reader_rows[0].index(header) + self.assertEqual(value, reader_rows[row_index][target_column]) def assertTaskProducesCSV(self, user, model, assert_fields_and_values): self._assertTaskProducesCSVBase(user, model, assert_fields_and_values) @@ -82,6 +85,7 @@ def assertPsuedoAsyncTaskWorks(self, model, job_id = ctx['job_id'] job = ExportJob.objects.get(pk=job_id) + self.assertCSVRowValue(job.outfile, 1, {assertion_field: assertion_value}) @@ -213,16 +217,18 @@ class UserExportsTest(UserExportsTestCase): def get_csv_data_with_base_assertions(self): resp = users_csv(make_request(), self.instance) - reader = csv.reader(resp) - # Skip BOM and entry line - next(reader) - next(reader) + # decode bytes object into string list required by the csv reader + str_rows = [line.decode('utf-8') for line in resp] + + # strip the BOM and entry line out + reader = csv.reader(str_rows[2:]) + # grab and strip the header header = next(reader) - data = [dict(list(zip(header, [x.decode('utf8') for x in row]))) - for row in reader] + data = (lambda h=header, reader=reader: + [dict(list(zip(h, [x for x in row]))) for row in reader])() commander, user1data, user2data = data self.assertEqual(commander['username'], self.commander.username) diff --git a/opentreemap/exporter/util.py b/opentreemap/exporter/util.py index 4db204e25..5b83de91f 100644 --- a/opentreemap/exporter/util.py +++ b/opentreemap/exporter/util.py @@ -4,11 +4,11 @@ def sanitize_unicode_value(value): # make sure every text value is of type 'str', coercing unicode if isinstance(value, str): - return value.encode("utf-8") - elif isinstance(value, str): return value + elif isinstance(value, int): + return str(value) else: - return str(value).encode("utf-8") + return value.decode("utf-8") # originally copied from, but now divergent from: diff --git a/opentreemap/importer/tasks.py b/opentreemap/importer/tasks.py index a757c668c..1a5b11409 100644 --- a/opentreemap/importer/tasks.py +++ b/opentreemap/importer/tasks.py @@ -22,7 +22,7 @@ def _create_rows_for_event(ie, csv_file): # so we can show progress. Caller does manual cleanup if necessary. reader = utf8_file_to_csv_dictreader(csv_file) - field_names = [f.strip().decode('utf-8') for f in reader.fieldnames + field_names = [f.strip() for f in reader.fieldnames if f.strip().lower() not in ie.ignored_fields()] ie.field_order = json.dumps(field_names) ie.save() diff --git a/opentreemap/importer/tests.py b/opentreemap/importer/tests.py index f199ae518..208aea991 100644 --- a/opentreemap/importer/tests.py +++ b/opentreemap/importer/tests.py @@ -461,7 +461,7 @@ def test_date_udf(self): i = self.mkrow(row) i.validate_row() self.assertHasError(i, errors.INVALID_UDF_VALUE, - data="[u'Test date must be formatted as YYYY-MM-DD']") + data="['Test date must be formatted as YYYY-MM-DD']") def test_choice_udf(self): UserDefinedFieldDefinition.objects.create( @@ -534,13 +534,13 @@ def test_multichoice_udf(self): i = self.mkrow(row) i.validate_row() self.assertHasError(i, errors.INVALID_UDF_VALUE, - data="[u'Test multichoice must be valid JSON']") + data="['Test multichoice must be valid JSON']") row['planting site: test multichoice'] = '"a","b"' i = self.mkrow(row) i.validate_row() self.assertHasError(i, errors.INVALID_UDF_VALUE, - data="[u'Test multichoice must be valid JSON']") + data="['Test multichoice must be valid JSON']") class SpeciesValidationTest(ValidationTest): diff --git a/opentreemap/importer/util.py b/opentreemap/importer/util.py index 6dec73631..da14db32c 100644 --- a/opentreemap/importer/util.py +++ b/opentreemap/importer/util.py @@ -3,7 +3,7 @@ import codecs import csv - +from io import StringIO def _clean_string(s): s = s.strip() @@ -34,7 +34,7 @@ def _as_utf8(f): def _guess_dialect_and_reset_read_pointer(f): - dialect = csv.Sniffer().sniff(_as_utf8(f).read(4096), delimiters=',\t') + dialect = csv.Sniffer().sniff(_as_utf8(f).read(4096).decode(), delimiters=',\t') f.seek(0) return dialect @@ -45,5 +45,11 @@ def utf8_file_to_csv_dictreader(f): # CSV standard "" to escape a double quote. Excel and LibreOffice # use this escape by default when saving as CSV. dialect.doublequote = True - return csv.DictReader(_as_utf8(f), + + # at this point, the binary filestream must be repackaged to the csv spec, + # that is an iterator of unicode strings. time limited a more elegant solution + # than allocating memory for up to two copies of the data but we anticipate having + # sufficient RAM + csv_body = StringIO(f.read().decode()) + return csv.DictReader(csv_body, dialect=dialect) diff --git a/opentreemap/importer/views.py b/opentreemap/importer/views.py index f8d03dddb..971740983 100644 --- a/opentreemap/importer/views.py +++ b/opentreemap/importer/views.py @@ -707,7 +707,7 @@ def process_csv(request, instance, import_type, **kwargs): filename = list(files.keys())[0] file_obj = files[filename] - file_obj = io.BytesIO(decode(file_obj.read()).encode('utf-8')) + file_obj = io.BytesIO((file_obj.read().encode('utf-8'))) owner = request.user ImportEventModel = get_import_event_model(import_type) diff --git a/opentreemap/otm1_migrator/model_processors.py b/opentreemap/otm1_migrator/model_processors.py index e6b4d65b8..fc863205f 100644 --- a/opentreemap/otm1_migrator/model_processors.py +++ b/opentreemap/otm1_migrator/model_processors.py @@ -3,7 +3,6 @@ import os import pytz -from exceptions import NotImplementedError from django.db.transaction import atomic from django.contrib.contenttypes.models import ContentType @@ -138,7 +137,7 @@ def process_userprofile(migration_rules, migration_event, photo_full_path = os.path.join(photo_basepath, photo_path) try: - photo_data = open(photo_full_path) + photo_data = open(photo_full_path, 'rb') except IOError: print("Failed to read photo %s ... SKIPPING USER %s %s" % (photo_full_path, user.id, user.username)) @@ -164,7 +163,7 @@ def save_treephoto(migration_rules, migration_event, treephoto_path, pk = models.UNBOUND_MODEL_ID else: image = open(os.path.join(treephoto_path, - model_dict['fields']['photo'])) + model_dict['fields']['photo']), 'rb') treephoto_obj.set_image(image) treephoto_obj.map_feature_id = (Tree .objects diff --git a/opentreemap/treemap/audit.py b/opentreemap/treemap/audit.py index 1da7b390e..6414c233d 100644 --- a/opentreemap/treemap/audit.py +++ b/opentreemap/treemap/audit.py @@ -530,7 +530,7 @@ def as_dict(self): def hash(self): values = ['%s:%s' % (k, v) for (k, v) in self.as_dict().items()] string = '|'.join(values).encode('utf-8') - return hashlib.md5(string).hexdigest() + return hashlib.md5(string.encode()).hexdigest() class UserTrackable(Dictable): @@ -1154,7 +1154,7 @@ def hash(self): string_to_hash = '%s:%s:%s' % (self._model_name, self.pk, audit_string) - return hashlib.md5(string_to_hash).hexdigest() + return hashlib.md5(string_to_hash.encode()).hexdigest() @classmethod def action_format_string_for_audit(clz, audit): diff --git a/opentreemap/treemap/images.py b/opentreemap/treemap/images.py index f2909e850..3ce4355a7 100644 --- a/opentreemap/treemap/images.py +++ b/opentreemap/treemap/images.py @@ -4,7 +4,7 @@ from PIL import Image import hashlib import os -from io import StringIO +from io import BytesIO from django.conf import settings from django.core.exceptions import ValidationError @@ -27,7 +27,7 @@ def _rotate_image_based_on_exif(img): def _get_file_for_image(image, filename, format): - temp = StringIO() + temp = BytesIO() image.save(temp, format=format) temp.seek(0) return SimpleUploadedFile(filename, temp.read(), diff --git a/opentreemap/treemap/instance.py b/opentreemap/treemap/instance.py index 3dd37c954..107694876 100644 --- a/opentreemap/treemap/instance.py +++ b/opentreemap/treemap/instance.py @@ -153,7 +153,7 @@ def create_from_geojson(cls, geojson): try: geojson_dict = json.loads(geojson) except ValueError as e: - raise ValidationError(e.message) + raise ValidationError(str(e)) if geojson_dict['type'] != 'FeatureCollection': raise ValidationError('GeoJSON must contain a FeatureCollection') @@ -430,11 +430,11 @@ def center(self): @property def geo_rev_hash(self): - return hashlib.md5(str(self.geo_rev)).hexdigest() + return hashlib.md5(str(self.geo_rev).encode()).hexdigest() @property def universal_rev_hash(self): - return hashlib.md5(str(self.universal_rev)).hexdigest() + return hashlib.md5(str(self.universal_rev).encode()).hexdigest() @property def center_lat_lng(self): diff --git a/opentreemap/treemap/models.py b/opentreemap/treemap/models.py index a0c0562f4..af5015bf6 100644 --- a/opentreemap/treemap/models.py +++ b/opentreemap/treemap/models.py @@ -360,7 +360,7 @@ def created(self): @property def email_hash(self): - return hashlib.sha512(self.email).hexdigest() + return hashlib.sha512(self.email.encode()).hexdigest() def dict(self): return {'id': self.pk, @@ -822,7 +822,7 @@ def hash(self): for feature in self.nearby_map_features(): string_to_hash += "," + str(feature.pk) - return hashlib.md5(string_to_hash).hexdigest() + return hashlib.md5(string_to_hash.encode()).hexdigest() def title(self): # Cast allows the map feature subclass to handle generating @@ -1202,7 +1202,7 @@ def hash(self): photos = [str(photo.pk) for photo in self.treephoto_set.all()] string_to_hash += ":" + ",".join(photos) - return hashlib.md5(string_to_hash).hexdigest() + return hashlib.md5(string_to_hash.encode()).hexdigest() def add_photo(self, image, user): tp = TreePhoto(tree=self, instance=self.instance) diff --git a/opentreemap/treemap/templatetags/form_extras.py b/opentreemap/treemap/templatetags/form_extras.py index 5ac187646..e4b3e986d 100644 --- a/opentreemap/treemap/templatetags/form_extras.py +++ b/opentreemap/treemap/templatetags/form_extras.py @@ -59,19 +59,19 @@ class Variable(Grammar): grammar = (G('"', WORD('^"'), '"') | G("'", WORD("^'"), "'") | WORD("a-zA-Z_", "a-zA-Z0-9_.")) - + grammar_whitespace_mode = 'optional' class Label(Grammar): grammar = (G('_("', WORD('^"'), '")') | G("_('", WORD("^'"), "')") | Variable) - + grammar_whitespace_mode = 'optional' class InlineEditGrammar(Grammar): grammar = (OR(G(OR("field", "create"), OPTIONAL(Label)), "search"), "from", Variable, OPTIONAL("for", Variable), OPTIONAL("in", Variable), "withtemplate", Variable, OPTIONAL("withhelp", Label)) - grammar_whitespace = True + grammar_whitespace_mode = 'optional' _inline_edit_parser = InlineEditGrammar.parser() @@ -186,8 +186,8 @@ def inline_edit_tag(tag, Node): """ def tag_parser(parser, token): try: - results = _inline_edit_parser.parse_string(token.contents, - reset=True, eof=True) + results = _inline_edit_parser.parse_text(token.contents, + reset=True, eof=True) except ParseError as e: raise template.TemplateSyntaxError( 'expected format: %s [{label}] from {model.property}' @@ -224,7 +224,7 @@ def _token_to_variable(token): elif token[0] == '"' and token[0] == token[-1] and len(token) >= 2: return token[1:-1] else: - return template.Variable(token) + return template.Variable(token.strip()) def _resolve_variable(variable, context): diff --git a/opentreemap/treemap/tests/__init__.py b/opentreemap/treemap/tests/__init__.py index 202856598..26681b3e8 100644 --- a/opentreemap/treemap/tests/__init__.py +++ b/opentreemap/treemap/tests/__init__.py @@ -2,7 +2,7 @@ import logging -from io import StringIO +from io import BytesIO import subprocess import shutil import tempfile @@ -365,7 +365,7 @@ def make_request(params=None, user=None, instance=None, extra = {} if body: - body_stream = StringIO(body) + body_stream = BytesIO(body.encode()) extra['wsgi.input'] = body_stream extra['CONTENT_LENGTH'] = len(body) @@ -430,7 +430,7 @@ def resource_path(self, name): return path def load_resource(self, name): - return open(self.resource_path(name)) + return open(self.resource_path(name), 'rb') def tearDown(self): shutil.rmtree(self.photoDir) diff --git a/opentreemap/treemap/views/map_feature.py b/opentreemap/treemap/views/map_feature.py index 31a9d0cd1..f3adbe8c5 100644 --- a/opentreemap/treemap/views/map_feature.py +++ b/opentreemap/treemap/views/map_feature.py @@ -371,7 +371,8 @@ def map_feature_hash(request, instance, feature_id, edit=False, tree_id=None): if request.user: pk = request.user.pk or '' - return hashlib.md5(feature.hash + ':' + str(pk)).hexdigest() + string_to_hash = feature.hash + ':' + str(pk) + return hashlib.md5(string_to_hash.encode()).hexdigest() @get_photo_context_and_errors diff --git a/opentreemap/treemap/views/tree.py b/opentreemap/treemap/views/tree.py index 99b10c94c..13d2b7861 100644 --- a/opentreemap/treemap/views/tree.py +++ b/opentreemap/treemap/views/tree.py @@ -132,4 +132,4 @@ def ecobenefits_hash(request, instance): string_to_hash = universal_rev + ":" + eco_str + ":" + map_features - return hashlib.md5(string_to_hash).hexdigest() + return hashlib.md5(string_to_hash.encode()).hexdigest() diff --git a/requirements.txt b/requirements.txt index e9341d382..3ea7becc8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ amqp==2.2.1 anyjson==0.3.3 billiard==3.5.0.3 boto==2.48.0 # http://docs.pythonboto.org/en/latest/releasenotes/v2.39.0.html -celery==4.1.0 +celery==4.3.0 certifi==2017.7.27.1 chardet==3.0.4 # https://docs.djangoproject.com/en/1.10/releases/#id2 @@ -19,7 +19,7 @@ django-redis==4.8.0 django-registration-redux==1.7 django-storages==1.6.5 django-threadedcomments==1.1 -django-tinsel==1.0.1 +django-tinsel==1.0.2 django-webpack-loader==0.5.0 # https://github.com/owais/django-webpack-loader/releases flake8==2.0 # rq.filter: ==2.0 gunicorn==19.7.1 # http://docs.gunicorn.org/en/stable/news.html