From 4b9b1a25c1dc90ca963c7f979155fa231485345b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Sun, 5 Jul 2020 13:45:33 +0200 Subject: [PATCH 01/38] Escaping strings with triple quotes --- escape_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escape_helpers.py b/escape_helpers.py index 5807d71..20eb4b6 100644 --- a/escape_helpers.py +++ b/escape_helpers.py @@ -5,7 +5,7 @@ def sparql_escape_string(obj): obj = str(obj) def replacer(a): return "\\"+a.group(0) - return '"' + re.sub(r'[\\\'"]', replacer, obj) + '"' + return '"""' + re.sub(r'[\\\'"]', replacer, obj) + '"""' def sparql_escape_datetime(obj): if isinstance(obj, datetime.datetime): From 9ea163bd7160d92776c8bef54bf01881ac33cdec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Tue, 7 Jul 2020 16:05:52 +0200 Subject: [PATCH 02/38] Merge python3-port into master --- Dockerfile | 6 +++--- README.md | 5 ++++- escape_helpers.py | 6 +++--- web.py | 10 +++++----- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9038654..04e5923 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -FROM python:2.7 -MAINTAINER Sam Landuydt "sam.landuydt@gmail.com" +FROM python:3 +MAINTAINER Michaël Dierick "michael@dierick.io" ENV APP_ENTRYPOINT web ENV LOG_LEVEL info @@ -13,7 +13,7 @@ ADD . /usr/src/app RUN ln -s /app /usr/src/app/ext \ && cd /usr/src/app \ - && pip install -r requirements.txt + && pip3 install -r requirements.txt ONBUILD ADD . /app/ ONBUILD RUN cd /app/ \ diff --git a/README.md b/README.md index 88b8422..c99c13b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Template for running Python microservices 3) Write the python requirements in a requirements.txt file. (Flask, SPARQLWrapper and rdflib are standard installed) -Create the entry point file and add methods with URL's. +Create the entry point file and add methods with URL's. The flask app is added to the python builtin and can be accessed by using the app variable, as shown in following example: @app.route("/exampleMethod") @@ -45,6 +45,7 @@ To use the template while developing your app, start a container in development docker run --volume /path/to/your/code:/app -e MODE=development + -p 80:80 -d semtech/mu-python-template Code changes will be automatically picked up by Flask. @@ -113,5 +114,7 @@ For example: Next to the `sparql_escape`, the template also provides a helper function per datatype that takes any value as parameter. E.g. `sparql_escape_uri("http://mu.semte.ch/application")`. +Next to the `sparql_escape`, the template also provides a helper function per datatype that takes any value as parameter. E.g. `sparql_escape_uri("http://mu.semte.ch/application")`. + ## Example There is one example method in the template: `GET /templateExample`. This method returns all triples in the triplestore from the SPARQL endpoint (beware for big databases!). diff --git a/escape_helpers.py b/escape_helpers.py index 20eb4b6..000ab57 100644 --- a/escape_helpers.py +++ b/escape_helpers.py @@ -16,7 +16,7 @@ def sparql_escape_datetime(obj): return '"{}"^^xsd:dateTime'.format(obj.isoformat()) except ValueError as e: # Failed casting to string or invalid dateTime format raise e - + def sparql_escape_date(obj): if isinstance(obj, datetime.date): return '"{}"^^xsd:date'.format(obj.isoformat()) @@ -25,7 +25,7 @@ def sparql_escape_date(obj): return '"{}"^^xsd:date'.format(obj.isoformat()) except ValueError as e: # Failed casting to string or invalid date format raise e - + def sparql_escape_time(obj): if isinstance(obj, datetime.time): obj = obj.replace(microsecond=0) # xsd doesn't support microseconds in ISO time format @@ -35,7 +35,7 @@ def sparql_escape_time(obj): return '"{}"^^xsd:time'.format(obj.isoformat()) except ValueError as e: # Failed casting to string or invalid time format raise e - + def sparql_escape_int(obj): if isinstance(obj, int): return '"{}"^^xsd:integer'.format(obj) diff --git a/web.py b/web.py index e6cc492..4267ed8 100644 --- a/web.py +++ b/web.py @@ -1,7 +1,7 @@ import flask import os import helpers -import __builtin__ +import builtins from escape_helpers import sparql_escape from rdflib.namespace import Namespace @@ -37,14 +37,14 @@ def query(): ## Start Application ## ####################### if __name__ == '__main__': - __builtin__.app = app - __builtin__.helpers = helpers - __builtin__.sparql_escape = sparql_escape + builtins.app = app + builtins.helpers = helpers + builtins.sparql_escape = sparql_escape app_file = os.environ.get('APP_ENTRYPOINT') f = open('/app/__init__.py', 'w+') f.close() try: - exec "from ext.app.%s import *" % app_file + exec("from ext.app.%s import *" % app_file) except Exception as e: helpers.log(str(e)) debug = True if (os.environ.get('MODE') == "development") else False From 9453db94e85d85b6725e97aa19020871c858b16f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Tue, 7 Jul 2020 16:40:52 +0200 Subject: [PATCH 03/38] Elif + return linting --- escape_helpers.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/escape_helpers.py b/escape_helpers.py index 000ab57..375e633 100644 --- a/escape_helpers.py +++ b/escape_helpers.py @@ -72,17 +72,16 @@ def replacer(a): def sparql_escape(obj): if isinstance(obj, str): return sparql_escape_string(obj) - elif isinstance(obj, datetime.datetime): + if isinstance(obj, datetime.datetime): return sparql_escape_datetime(obj) - elif isinstance(obj, datetime.date): + if isinstance(obj, datetime.date): return sparql_escape_date(obj) - elif isinstance(obj, datetime.time): + if isinstance(obj, datetime.time): return sparql_escape_time(obj) - elif isinstance(obj, int): + if isinstance(obj, int): return sparql_escape_int(obj) - elif isinstance(obj, float): + if isinstance(obj, float): return sparql_escape_float(obj) - elif isinstance(obj, bool): + if isinstance(obj, bool): return sparql_escape_bool(obj) - else: - return "" + return "" From ec8c7b83a5de3d995c776c078abfdc702c2ae13c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Tue, 7 Jul 2020 16:41:39 +0200 Subject: [PATCH 04/38] Lambda functions for regex replacement --- escape_helpers.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/escape_helpers.py b/escape_helpers.py index 375e633..26e9570 100644 --- a/escape_helpers.py +++ b/escape_helpers.py @@ -3,9 +3,7 @@ def sparql_escape_string(obj): obj = str(obj) - def replacer(a): - return "\\"+a.group(0) - return '"""' + re.sub(r'[\\\'"]', replacer, obj) + '"""' + return '"""' + re.sub(r'[\\\'"]', lambda s: "\\" + s.group(0), obj) + '"""' def sparql_escape_datetime(obj): if isinstance(obj, datetime.datetime): @@ -65,9 +63,7 @@ def sparql_escape_bool(obj): def sparql_escape_uri(obj): obj = str(obj) - def replacer(a): - return "\\"+a.group(0) - return '<' + re.sub(r'[\\\'"]', replacer, obj) + '>' + return '<' + re.sub(r'[\\\'"]', lambda s: "\\" + s.group(0), obj) + '>' def sparql_escape(obj): if isinstance(obj, str): From 219d2bbe7189f685f54232e1fdd4ec96fc5330bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Tue, 7 Jul 2020 17:05:09 +0200 Subject: [PATCH 05/38] Warn about cast instead of useless re-throwing exception + fix bool esc --- escape_helpers.py | 72 +++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 40 deletions(-) diff --git a/escape_helpers.py b/escape_helpers.py index 26e9570..4f0a362 100644 --- a/escape_helpers.py +++ b/escape_helpers.py @@ -1,65 +1,57 @@ import datetime import re +from warnings import warn def sparql_escape_string(obj): - obj = str(obj) + if not isinstance(obj, str): + warn("You are escaping something that isn't a string with \ + the 'sparql_escape_string'-method. Implicit casting will occurr.") + obj = str(obj) return '"""' + re.sub(r'[\\\'"]', lambda s: "\\" + s.group(0), obj) + '"""' def sparql_escape_datetime(obj): - if isinstance(obj, datetime.datetime): - obj = obj.replace(microsecond=0) # xsd doesn't support microseconds in ISO time format - return '"{}"^^xsd:dateTime'.format(obj.isoformat()) - try: + if not isinstance(obj, datetime.datetime): + warn("You are escaping something that isn't a datetime with \ + the 'sparql_escape_datetime'-method. Implicit casting will occurr.") obj = datetime.datetime.strptime(str(obj), "%Y-%m-%dT%H:%M:%S") - return '"{}"^^xsd:dateTime'.format(obj.isoformat()) - except ValueError as e: # Failed casting to string or invalid dateTime format - raise e + obj = obj.replace(microsecond=0) # xsd doesn't support microseconds in ISO time format + return '"{}"^^xsd:dateTime'.format(obj.isoformat()) def sparql_escape_date(obj): - if isinstance(obj, datetime.date): - return '"{}"^^xsd:date'.format(obj.isoformat()) - try: + if not isinstance(obj, datetime.date): + warn("You are escaping something that isn't a date with \ + the 'sparql_escape_date'-method. Implicit casting will occurr.") obj = datetime.datetime.strptime(str(obj), "%Y-%m-%d").date() - return '"{}"^^xsd:date'.format(obj.isoformat()) - except ValueError as e: # Failed casting to string or invalid date format - raise e + return '"{}"^^xsd:date'.format(obj.isoformat()) def sparql_escape_time(obj): - if isinstance(obj, datetime.time): - obj = obj.replace(microsecond=0) # xsd doesn't support microseconds in ISO time format - return '"{}"^^xsd:time'.format(obj.isoformat()) - try: + if not isinstance(obj, datetime.time): + warn("You are escaping something that isn't a time with \ + the 'sparql_escape_time'-method. Implicit casting will occurr.") obj = datetime.datetime.strptime(str(obj), "%H:%M:%S").time() - return '"{}"^^xsd:time'.format(obj.isoformat()) - except ValueError as e: # Failed casting to string or invalid time format - raise e + obj = obj.replace(microsecond=0) # xsd doesn't support microseconds in ISO time format + return '"{}"^^xsd:time'.format(obj.isoformat()) def sparql_escape_int(obj): - if isinstance(obj, int): - return '"{}"^^xsd:integer'.format(obj) - try: + if not isinstance(obj, int): + warn("You are escaping something that isn't an int with \ + the 'sparql_escape_int'-method. Implicit casting will occurr.") obj = str(int(obj)) - return '"{}"^^xsd:integer'.format(obj) - except ValueError as e: # Failed casting - raise e + return '"{}"^^xsd:integer'.format(obj) def sparql_escape_float(obj): - if isinstance(obj, float): - return '"{}"^^xsd:float'.format(obj) - try: + if not isinstance(obj, int): + warn("You are escaping something that isn't a float with \ + the 'sparql_escape_float'-method. Implicit casting will occurr.") obj = str(float(obj)) - return '"{}"^^xsd:float'.format(obj) - except ValueError as e: # Failed casting - raise e + return '"{}"^^xsd:float'.format(obj) def sparql_escape_bool(obj): - if isinstance(obj, bool): - return '"{}"^^xsd:boolean'.format(obj) - try: - obj = str(bool(obj)) - return '"{}"^^xsd:boolean'.format(obj) - except ValueError as e: # Failed casting - raise e + if not isinstance(obj, bool): + warn("You are escaping something that isn't a bool with \ + the 'sparql_escape_bool'-method. Implicit casting will occurr.") + obj = bool(obj) + return '"{}"^^xsd:boolean'.format("true" if obj else "false") def sparql_escape_uri(obj): obj = str(obj) From 0bd1f3fb4df5a5af6ac5d8547f3cebbddb29915d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Tue, 7 Jul 2020 17:05:26 +0200 Subject: [PATCH 06/38] linting + default to string --- escape_helpers.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/escape_helpers.py b/escape_helpers.py index 4f0a362..dd31450 100644 --- a/escape_helpers.py +++ b/escape_helpers.py @@ -59,17 +59,20 @@ def sparql_escape_uri(obj): def sparql_escape(obj): if isinstance(obj, str): - return sparql_escape_string(obj) - if isinstance(obj, datetime.datetime): - return sparql_escape_datetime(obj) - if isinstance(obj, datetime.date): - return sparql_escape_date(obj) - if isinstance(obj, datetime.time): - return sparql_escape_time(obj) - if isinstance(obj, int): - return sparql_escape_int(obj) - if isinstance(obj, float): - return sparql_escape_float(obj) - if isinstance(obj, bool): - return sparql_escape_bool(obj) - return "" + escaped_val = sparql_escape_string(obj) + elif isinstance(obj, datetime.datetime): + escaped_val = sparql_escape_datetime(obj) + elif isinstance(obj, datetime.date): + escaped_val = sparql_escape_date(obj) + elif isinstance(obj, datetime.time): + escaped_val = sparql_escape_time(obj) + elif isinstance(obj, int): + escaped_val = sparql_escape_int(obj) + elif isinstance(obj, float): + escaped_val = sparql_escape_float(obj) + elif isinstance(obj, bool): + escaped_val = sparql_escape_bool(obj) + else: + warn("Unknown escape type '{}'. Escaping as string".format(type(obj))) + escaped_val = sparql_escape_string(obj) + return escaped_val From 19f858349178d3254072053bbdf8a45ab6123a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Tue, 7 Jul 2020 17:07:46 +0200 Subject: [PATCH 07/38] Pin to python 3.8 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 04e5923..f192a2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.8 MAINTAINER Michaël Dierick "michael@dierick.io" ENV APP_ENTRYPOINT web From 9528514a2bb3ab1c6c64c861dadf01106bd59df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 09:54:42 +0200 Subject: [PATCH 08/38] Remove circular dependency --- helpers.py | 6 +++--- web.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/helpers.py b/helpers.py index 34b3857..81142c8 100644 --- a/helpers.py +++ b/helpers.py @@ -3,12 +3,12 @@ import logging import os import sys -from web import graph from flask import jsonify from rdflib.namespace import DC from escape_helpers import sparql_escape from SPARQLWrapper import SPARQLWrapper, JSON +MU_APPLICATION_GRAPH = os.environ.get('MU_APPLICATION_GRAPH') def generate_uuid(): """Generates a unique user id based the host ID and current time""" @@ -90,7 +90,7 @@ def update(the_query): def update_modified(subject, modified=datetime.datetime.now()): """Executes a SPARQL query to update the modification date of the given subject URI (string). The default date is now.""" - query = " WITH <%s> " % graph + query = " WITH <%s> " % MU_APPLICATION_GRAPH query += " DELETE {" query += " < %s > < %s > %s ." % (subject, DC.Modified, sparql_escape(modified)) query += " }" @@ -100,7 +100,7 @@ def update_modified(subject, modified=datetime.datetime.now()): update(query) query = " INSERT DATA {" - query += " GRAPH <%s> {" % graph + query += " GRAPH <%s> {" % MU_APPLICATION_GRAPH query += " <%s> <%s> %s ." % (subject, DC.Modified, sparql_escape(modified)) query += " }" query += " }" diff --git a/web.py b/web.py index 4267ed8..1ad2a4f 100644 --- a/web.py +++ b/web.py @@ -30,7 +30,6 @@ def query(): mu_core = Namespace('http://mu.semte.ch/vocabularies/core/') mu_ext = Namespace('http://mu.semte.ch/vocabularies/ext/') -graph = os.environ.get('MU_APPLICATION_GRAPH') SERVICE_RESOURCE_BASE = 'http://mu.semte.ch/services/' ####################### From dc1a2e37c32e6547577bdd7e39ab2e447c8510da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 11:37:52 +0200 Subject: [PATCH 09/38] Switch base image for production WSGI server --- Dockerfile | 14 +++++++++++--- start.sh | 8 ++++++++ web.py | 22 ++++++++++++---------- 3 files changed, 31 insertions(+), 13 deletions(-) create mode 100644 start.sh diff --git a/Dockerfile b/Dockerfile index f192a2d..b5e945c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,16 @@ -FROM python:3.8 +FROM tiangolo/meinheld-gunicorn:python3.8 MAINTAINER Michaël Dierick "michael@dierick.io" +# Gunicorn Docker config +ENV MODULE_NAME web +ENV PYTHONPATH "/usr/src/app" + +# Overrides the start.sh used in `tiangolo/meinheld-gunicorn` +COPY ./start.sh /start.sh +RUN chmod +x /start.sh + + +# Template config ENV APP_ENTRYPOINT web ENV LOG_LEVEL info ENV MU_SPARQL_ENDPOINT 'http://database:8890/sparql' @@ -19,6 +29,4 @@ ONBUILD ADD . /app/ ONBUILD RUN cd /app/ \ && if [ -f requirements.txt ]; then pip install -r requirements.txt; fi -EXPOSE 80 -CMD python web.py diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..59b2660 --- /dev/null +++ b/start.sh @@ -0,0 +1,8 @@ +#! /usr/bin/env sh +set -e + +if [ $MODE = "development" ]; then + exec python web.py +else + exec gunicorn -k egg:meinheld#gunicorn_worker -c "$GUNICORN_CONF" "$APP_MODULE" +fi diff --git a/web.py b/web.py index 1ad2a4f..f63b07d 100644 --- a/web.py +++ b/web.py @@ -5,6 +5,7 @@ from escape_helpers import sparql_escape from rdflib.namespace import Namespace +# WSGI variable name used by the server app = flask.Flask(__name__) #################### @@ -32,19 +33,20 @@ def query(): SERVICE_RESOURCE_BASE = 'http://mu.semte.ch/services/' +builtins.app = app +builtins.helpers = helpers +builtins.sparql_escape = sparql_escape +app_file = os.environ.get('APP_ENTRYPOINT') +f = open('/app/__init__.py', 'w+') +f.close() +try: + exec("from ext.app.%s import *" % app_file) +except Exception as e: + helpers.log(str(e)) + ####################### ## Start Application ## ####################### if __name__ == '__main__': - builtins.app = app - builtins.helpers = helpers - builtins.sparql_escape = sparql_escape - app_file = os.environ.get('APP_ENTRYPOINT') - f = open('/app/__init__.py', 'w+') - f.close() - try: - exec("from ext.app.%s import *" % app_file) - except Exception as e: - helpers.log(str(e)) debug = True if (os.environ.get('MODE') == "development") else False app.run(debug=debug, host='0.0.0.0', port=80) From dda6b47814a8d9f4b89d148885b05f4687feec80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 11:50:07 +0200 Subject: [PATCH 10/38] Remove incorrect statement regarding xsd date formats --- escape_helpers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/escape_helpers.py b/escape_helpers.py index dd31450..46e931f 100644 --- a/escape_helpers.py +++ b/escape_helpers.py @@ -14,8 +14,7 @@ def sparql_escape_datetime(obj): warn("You are escaping something that isn't a datetime with \ the 'sparql_escape_datetime'-method. Implicit casting will occurr.") obj = datetime.datetime.strptime(str(obj), "%Y-%m-%dT%H:%M:%S") - obj = obj.replace(microsecond=0) # xsd doesn't support microseconds in ISO time format - return '"{}"^^xsd:dateTime'.format(obj.isoformat()) + return '"{}"^^xsd:dateTime'.format(obj.isoformat(timespec="seconds")) def sparql_escape_date(obj): if not isinstance(obj, datetime.date): @@ -29,8 +28,7 @@ def sparql_escape_time(obj): warn("You are escaping something that isn't a time with \ the 'sparql_escape_time'-method. Implicit casting will occurr.") obj = datetime.datetime.strptime(str(obj), "%H:%M:%S").time() - obj = obj.replace(microsecond=0) # xsd doesn't support microseconds in ISO time format - return '"{}"^^xsd:time'.format(obj.isoformat()) + return '"{}"^^xsd:time'.format(obj.isoformat(timespec="seconds")) def sparql_escape_int(obj): if not isinstance(obj, int): From 6a47ba531530c1d55e6d653aa8ddf063390967ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 11:53:37 +0200 Subject: [PATCH 11/38] simplify date & time escaping --- escape_helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/escape_helpers.py b/escape_helpers.py index 46e931f..9af99d3 100644 --- a/escape_helpers.py +++ b/escape_helpers.py @@ -13,21 +13,21 @@ def sparql_escape_datetime(obj): if not isinstance(obj, datetime.datetime): warn("You are escaping something that isn't a datetime with \ the 'sparql_escape_datetime'-method. Implicit casting will occurr.") - obj = datetime.datetime.strptime(str(obj), "%Y-%m-%dT%H:%M:%S") + obj = datetime.datetime.fromisoformat(str(obj)) return '"{}"^^xsd:dateTime'.format(obj.isoformat(timespec="seconds")) def sparql_escape_date(obj): if not isinstance(obj, datetime.date): warn("You are escaping something that isn't a date with \ the 'sparql_escape_date'-method. Implicit casting will occurr.") - obj = datetime.datetime.strptime(str(obj), "%Y-%m-%d").date() + obj = datetime.date.fromisoformat(str(obj)) return '"{}"^^xsd:date'.format(obj.isoformat()) def sparql_escape_time(obj): if not isinstance(obj, datetime.time): warn("You are escaping something that isn't a time with \ the 'sparql_escape_time'-method. Implicit casting will occurr.") - obj = datetime.datetime.strptime(str(obj), "%H:%M:%S").time() + obj = datetime.time.fromisoformat(str(obj)) return '"{}"^^xsd:time'.format(obj.isoformat(timespec="seconds")) def sparql_escape_int(obj): From 0ac3da8142ac5b1eeef7794e0ff66c9dbeaa207b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 11:58:05 +0200 Subject: [PATCH 12/38] Added a note regarding logging --- helpers.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/helpers.py b/helpers.py index 81142c8..ab35872 100644 --- a/helpers.py +++ b/helpers.py @@ -10,16 +10,14 @@ MU_APPLICATION_GRAPH = os.environ.get('MU_APPLICATION_GRAPH') -def generate_uuid(): - """Generates a unique user id based the host ID and current time""" - return str(uuid.uuid1()) - - -log_levels = {'DEBUG': logging.DEBUG, - 'INFO': logging.INFO, - 'WARNING': logging.WARNING, - 'ERROR': logging.ERROR, - 'CRITICAL': logging.CRITICAL} +# TODO: Figure out how logging works when production uses multiple workers +log_levels = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRITICAL': logging.CRITICAL +} log_dir = '/logs' if not os.path.exists(log_dir): os.makedirs(log_dir) thelogger = logging.getLogger('') @@ -29,6 +27,11 @@ def generate_uuid(): consoleHandler = logging.StreamHandler(stream=sys.stdout)# or stderr? thelogger.addHandler(consoleHandler) +def generate_uuid(): + """Generates a unique user id based the host ID and current time""" + return str(uuid.uuid1()) + + def log(msg, *args, **kwargs): """write a log message to the log file. Logs are written to the `/logs` directory in the docker container.""" From 0282b265f3e11e98702059c3f00af521ae9d0034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 15:58:07 +0200 Subject: [PATCH 13/38] Started reworking README. Quickstart in a few steps --- README.md | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index c99c13b..e0989e1 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,41 @@ # Mu Python template -Template for running Python microservices +Template for [mu.semte.ch](http://mu.semte.ch)-microservices written in Python3. Based on the [Flask](https://palletsprojects.com/p/flask/)-framework. -## Using the template +## Quickstart -1) Extend the `semtech/mu-python-template` and set a maintainer. +Create a `Dockerfile` which extends the `semtech/mu-python-template`-image and set a maintainer. +```docker +FROM semtech/mu-python-template +MAINTAINER Sam Landuydt +``` -2) Configure your entrypoint through the environment variable `APP_ENTRYPOINT` (default: `web.py`). +Create a `web.py` entrypoint-file. (naming of the entrypoint can be configured through `APP_ENTRYPOINT`) +```python +@app.route("/hello") +def hello(): + return "Hello from the mu-python-template!" +``` -3) Write the python requirements in a requirements.txt file. (Flask, SPARQLWrapper and rdflib are standard installed) +Build the Docker-image for your service +```sh +docker build -t my-python-service . +``` -Create the entry point file and add methods with URL's. -The flask app is added to the python builtin and can be accessed by using the app variable, as shown in following example: +Run your service +```sh +docker run -p 8080:80 +``` - @app.route("/exampleMethod") - def exampleMethod(): - return example +You now should be able to access your service's endpoint +```sh +curl localhost:8080/hello +``` -## Example Dockerfile +## Dependencies + +If your service needs external libraries other than the ones already provided by the template (Flask, SPARQLWrapper and rdflib), you can specify those in a [`requirements.txt`](https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format)-file. The template will take care of installing them when you build your Docker image. - FROM semtech/mu-python-template - MAINTAINER Sam Landuydt - # ONBUILD of mu-python-template takes care of everything ## Configuration From 39fb82fe2966fd64c64a9a4e9e0832aff59bcb96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 18:25:40 +0200 Subject: [PATCH 14/38] Rework helper method documentation --- README.md | 136 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 77 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index e0989e1..c706d45 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Template for [mu.semte.ch](http://mu.semte.ch)-microservices written in Python3. Create a `Dockerfile` which extends the `semtech/mu-python-template`-image and set a maintainer. ```docker FROM semtech/mu-python-template -MAINTAINER Sam Landuydt +LABEL maintainer="sam.landuydt@gmail.com" ``` Create a `web.py` entrypoint-file. (naming of the entrypoint can be configured through `APP_ENTRYPOINT`) @@ -32,103 +32,121 @@ You now should be able to access your service's endpoint curl localhost:8080/hello ``` -## Dependencies +## Developing a microservice using the template + +### Dependencies If your service needs external libraries other than the ones already provided by the template (Flask, SPARQLWrapper and rdflib), you can specify those in a [`requirements.txt`](https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format)-file. The template will take care of installing them when you build your Docker image. +### Development mode -## Configuration +By leveraging Dockers' [bind-mount](https://docs.docker.com/storage/bind-mounts/), you can mount your application code into an existing service image. This spares you from building a new image to test each change. On top of that, you can configure the environment variable `MODE` to `development`. That enables live-reloading of the server, so it immediately updates when you save a file. -The template supports the following environment variables: +example docker-compose parameters: +```yml + environment: + MODE: "development" + volumes: + - /home/my/code/my-python-service:/app +``` -- `MU_SPARQL_ENDPOINT` is used to configure the SPARQL endpoint. +### Helper methods - - By default this is set to `http://database:8890/sparql`. In that case the triple store used in the backend should be linked to the microservice container as `database`. +The template provides the user with several helper methods. They aim to give you a step ahead for: +- logging +- JSONAPI-compliancy +- SPARQL querying -- `MU_APPLICATION_GRAPH` specifies the graph in the triple store the microservice will work in. +The below helpers can be imported from the `helpers` module. For example: +```py +from helpers import * +``` +Available functions: +#### log(msg) - - By default this is set to `http://mu.semte.ch/application`. The graph name can be used in the service via `settings.graph`. +Works exactly the same as the [log](https://docs.python.org/3/library/logging.html#logging.log) method from pythons' logging module. +Logs are written to the /logs directory in the docker container. +#### generate_uuid() -- `MU_SPARQL_TIMEOUT` is used to configure the timeout (in seconds) for SPARQL queries. +Generate a random UUID (String). -## Develop a microservice using the template +#### session_id_header(request) -To use the template while developing your app, start a container in development mode with your code folder on the host machine mounted in `/app`: +Get the session id from the HTTP request headers. - docker run --volume /path/to/your/code:/app - -e MODE=development - -p 80:80 - -d semtech/mu-python-template +#### rewrite_url_header(request) -Code changes will be automatically picked up by Flask. +Get the rewrite URL from the HTTP request headers. -## Helper methods -The template provides the user with several helper methods. Most helpers can be used by calling: "helpers.", except the sparql_escape helper: "sparql_escape(var)". +#### validate_json_api_content_type(request) -### log(msg) +Validate whether the Content-Type header contains the JSONAPI `content-type`-header. Returns a 400 otherwise. -The template provides a log object to the user for logging. Just do log("Hello world"). -The log level can be set through the LOG_LEVEL environment variable - (default: info, values: debug, info, warning, error, critical). +#### validate_resource_type(expected_type, data) -Logs are written to the /logs directory in the docker container. +Validate whether the type specified in the JSONAPI data is equal to the expected type. Returns a 409 otherwise. -### generate_uuid() +#### error(title, status = 400) -Generate a random UUID (String). +Returns a JSONAPI compliant error response with the given status code (default: 400). -### session_id_header(request) +#### query(query) -Get the session id from the HTTP request headers. +Executes the given SPARQL select/ask/construct query. -### rewrite_url_header(request) +#### update(query) -Get the rewrite URL from the HTTP request headers. +Executes the given SPARQL update query. -### validate_json_api_content_type(request) -Validate whether the Content-Type header contains the JSONAPI Content-Type. Returns a 400 otherwise. +The template provides one other helper module, being the `escape_helpers`-module. It contains functions for SPARQL query-escaping. Example import: +```py +from escape_helpers import * +``` -### validate_resource_type(expected_type, data) + Available functions: +#### sparql_escape ; sparql_escape_{string|uri|date|datetime|time|bool|int|float}(value) -Validate whether the type specified in the JSON data is equal to the expected type. Returns a 409 otherwise. +Converts the given object to a SPARQL-safe RDF object string with the right RDF-datatype. +This functions should be used especially when inserting user-input to avoid SPARQL-injection. -### error(title, status = 400) +Separate functions are available for different python datatypes, the `sparql_escape` function however can automatically select the right method to use, for following Python datatypes: -Returns a JSONAPI compliant error response with the given status code (default: 400). +- `str` +- `int` +- `float` +- `datetime.datetime` +- `datetime.date` +- `datetime.time` +- `boolean` -### query(query) +The `sparql_escape_uri`-function can be used for escaping URI's. -Executes the given SPARQL select/ask/construct query. -### update(query) +## Configuration -Executes the given SPARQL update query. +The template supports the following environment variables: -### update_modified(subject, modified = datetime.now()) +- `MU_SPARQL_ENDPOINT` is used to configure the SPARQL endpoint. -Executes a SPARQL query to update the modification date of the given subject URI (string). -The date defaults to now. + - By default this is set to `http://database:8890/sparql`. In that case the triple store used in the backend should be linked to the microservice container as `database`. -### sparql_escape ; sparql_escape_{string|uri|date|datetime|time|bool|int|float}(value) -This method can be used to avoid SPARQL injection by escaping user input while constructing a SPARQL query. -The method checks the type of the given variable and returns the correct object string format, -depending on the type of the object. Current supported variables are: `datetime.time`, `datetime.date`, `str`, `int`, `float` and `boolean`. -For example: - query = " INSERT DATA {" - query += " GRAPH {" - query += " < %s > a ;" % user_uri - query += " %s ;" % sparql_escape(name) - query += " %s ." % sparql_escape(date) - query += " }" - query += " }" - -Next to the `sparql_escape`, the template also provides a helper function per datatype that takes any value as parameter. E.g. `sparql_escape_uri("http://mu.semte.ch/application")`. +- `MU_APPLICATION_GRAPH` specifies the graph in the triple store the microservice will work in. -Next to the `sparql_escape`, the template also provides a helper function per datatype that takes any value as parameter. E.g. `sparql_escape_uri("http://mu.semte.ch/application")`. + - By default this is set to `http://mu.semte.ch/application`. The graph name can be used in the service via `settings.graph`. + + +- `MU_SPARQL_TIMEOUT` is used to configure the timeout (in seconds) for SPARQL queries. + + +To use the template while developing your app, start a container in development mode with your code folder on the host machine mounted in `/app`: + + docker run --volume /path/to/your/code:/app + -e MODE=development + -p 80:80 + -d semtech/mu-python-template -## Example -There is one example method in the template: `GET /templateExample`. This method returns all triples in the triplestore from the SPARQL endpoint (beware for big databases!). +). From f2bee6e9007e1d64efb3ac6da1f1b783a4929b32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 18:38:03 +0200 Subject: [PATCH 15/38] Add and modify some documentation regarding deployment --- README.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c706d45..15d2b94 100644 --- a/README.md +++ b/README.md @@ -124,10 +124,21 @@ Separate functions are available for different python datatypes, the `sparql_esc The `sparql_escape_uri`-function can be used for escaping URI's. +## Deployment -## Configuration +Example snippet for adding a service to a docker-compose stack: +```yml +my-python: + image: my-python-service + environment: + LOG_LEVEL: "debug" +``` + +### Environment variables -The template supports the following environment variables: +- `LOG_LEVEL` takes the same options as defined in the Python [logging](https://docs.python.org/3/library/logging.html#logging-levels) module. + +- `MODE` to specify the deployment mode. Can be `development` as well as `production`. Defaults to `production` - `MU_SPARQL_ENDPOINT` is used to configure the SPARQL endpoint. @@ -142,11 +153,6 @@ The template supports the following environment variables: - `MU_SPARQL_TIMEOUT` is used to configure the timeout (in seconds) for SPARQL queries. -To use the template while developing your app, start a container in development mode with your code folder on the host machine mounted in `/app`: - - docker run --volume /path/to/your/code:/app - -e MODE=development - -p 80:80 - -d semtech/mu-python-template +### Production -). +For hosting the app in a production setting, the template depends on [meinheld-gunicorn-docker](https://github.com/tiangolo/meinheld-gunicorn-docker). All [environment variables](https://github.com/tiangolo/meinheld-gunicorn-docker#environment-variables) used by meinheld-gunicorn can be used to configure your service as well. From 562751311c134bf29f8b54df67091775d278ad30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 18:43:36 +0200 Subject: [PATCH 16/38] More verbosity about /app --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 15d2b94..f7e37ae 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ If your service needs external libraries other than the ones already provided by ### Development mode -By leveraging Dockers' [bind-mount](https://docs.docker.com/storage/bind-mounts/), you can mount your application code into an existing service image. This spares you from building a new image to test each change. On top of that, you can configure the environment variable `MODE` to `development`. That enables live-reloading of the server, so it immediately updates when you save a file. +By leveraging Dockers' [bind-mount](https://docs.docker.com/storage/bind-mounts/), you can mount your application code into an existing service image. This spares you from building a new image to test each change. Just mount your services' folder to the containers' `/app`. On top of that, you can configure the environment variable `MODE` to `development`. That enables live-reloading of the server, so it immediately updates when you save a file. example docker-compose parameters: ```yml From 10f3b70f7288fc8a9b820de1ff85c52fd8c85fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 23:54:38 +0200 Subject: [PATCH 17/38] Ideomatic module import --- Dockerfile | 1 + web.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b5e945c..f008ec7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ RUN ln -s /app /usr/src/app/ext \ && pip3 install -r requirements.txt ONBUILD ADD . /app/ +ONBUILD RUN touch /app/__init__.py ONBUILD RUN cd /app/ \ && if [ -f requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/web.py b/web.py index f63b07d..d7a08ee 100644 --- a/web.py +++ b/web.py @@ -1,6 +1,7 @@ import flask import os import helpers +from importlib import import_module import builtins from escape_helpers import sparql_escape from rdflib.namespace import Namespace @@ -36,11 +37,12 @@ def query(): builtins.app = app builtins.helpers = helpers builtins.sparql_escape = sparql_escape + +# Import the app from the service consuming the template app_file = os.environ.get('APP_ENTRYPOINT') -f = open('/app/__init__.py', 'w+') -f.close() try: - exec("from ext.app.%s import *" % app_file) + module_path = 'ext.app.{}'.format(app_file) + import_module(module_path) except Exception as e: helpers.log(str(e)) From 75cd13076a41506a1dfa029425cb0911ff32b7a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 23:55:16 +0200 Subject: [PATCH 18/38] Reshuffled imports --- web.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web.py b/web.py index d7a08ee..bf574e5 100644 --- a/web.py +++ b/web.py @@ -1,11 +1,13 @@ -import flask import os -import helpers from importlib import import_module import builtins -from escape_helpers import sparql_escape + +import flask from rdflib.namespace import Namespace +import helpers +from escape_helpers import sparql_escape + # WSGI variable name used by the server app = flask.Flask(__name__) From 84443e9dc324299dd8e279ca3c906a123d810fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 25 Sep 2020 23:55:27 +0200 Subject: [PATCH 19/38] Removed example method --- web.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/web.py b/web.py index bf574e5..f500348 100644 --- a/web.py +++ b/web.py @@ -11,22 +11,6 @@ # WSGI variable name used by the server app = flask.Flask(__name__) -#################### -## Example method ## -#################### - -@app.route('/templateExample/') -def query(): - """Example query: Returns all the triples in the application graph in a JSON - format.""" - q = " SELECT *" - q += " WHERE{" - q += " GRAPH {" - q += " ?s ?p ?o" - q += " }" - q += " }" - return flask.jsonify(helpers.query(q)) - ################## ## Vocabularies ## ################## From 4d22fc25f1476a861445e7dcac947cc83bbd7632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Thu, 29 Oct 2020 10:27:58 +0100 Subject: [PATCH 20/38] Added documentation on constructing sparql queries --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index f7e37ae..29628c3 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,30 @@ Separate functions are available for different python datatypes, the `sparql_esc The `sparql_escape_uri`-function can be used for escaping URI's. +### Writing SPARQL Queries + +The template itself is unopinionated when it comes to constructing SPARQL-queries. However, since Python's most common string formatting methods aren't a great fit for SPARQL queries, we hereby want to provide an example on how to construct a query based on [template strings](https://docs.python.org/3.8/library/string.html#template-strings) while keeping things readable. + +```py +from string import Template +from helpers import query +from escape_helpers import sparql_escape_uri + +my_person = "http://example.com/me" +query_template = Template(""" +PREFIX mu: +PREFIX foaf: + +SELECT ?name +WHERE { + $person a foaf:Person ; + foaf:firstName ?name . +} +""") +query_string = query_template.substitute(person=sparql_escape_uri(my_person)) +query_result = query(query_string) +``` + ## Deployment Example snippet for adding a service to a docker-compose stack: From dd03797129f31fbdf3b0c734b1361ca187858f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Thu, 29 Oct 2020 11:36:42 +0100 Subject: [PATCH 21/38] Expose & document the logger used by the template --- README.md | 5 +++-- helpers.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 29628c3..4fc74f2 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,9 @@ from helpers import * Available functions: #### log(msg) -Works exactly the same as the [log](https://docs.python.org/3/library/logging.html#logging.log) method from pythons' logging module. -Logs are written to the /logs directory in the docker container. +Works exactly the same as the [logging.info](https://docs.python.org/3/library/logging.html#logging.info) method from pythons' logging module. +Logs are written to the /logs directory in the docker container. +Note that the `helpers` module also exposes `logger`, which is the [logger instance](https://docs.python.org/3/library/logging.html#logger-objects) used by the template. The methods provided by this instance can be used for more fine-grained logging. #### generate_uuid() diff --git a/helpers.py b/helpers.py index ab35872..628ae4f 100644 --- a/helpers.py +++ b/helpers.py @@ -20,12 +20,12 @@ } log_dir = '/logs' if not os.path.exists(log_dir): os.makedirs(log_dir) -thelogger = logging.getLogger('') -thelogger.setLevel(log_levels.get(os.environ.get('LOG_LEVEL').upper())) +logger = logging.getLogger('MU_PYTHON_TEMPLATE_LOGGER') +logger.setLevel(log_levels.get(os.environ.get('LOG_LEVEL').upper())) fileHandler = logging.FileHandler("{0}/{1}.log".format(log_dir, 'logs')) -thelogger.addHandler(fileHandler) +logger.addHandler(fileHandler) consoleHandler = logging.StreamHandler(stream=sys.stdout)# or stderr? -thelogger.addHandler(consoleHandler) +logger.addHandler(consoleHandler) def generate_uuid(): """Generates a unique user id based the host ID and current time""" @@ -35,7 +35,7 @@ def generate_uuid(): def log(msg, *args, **kwargs): """write a log message to the log file. Logs are written to the `/logs` directory in the docker container.""" - thelogger.info(msg, *args, **kwargs) + return logger.info(msg, *args, **kwargs) def session_id_header(request): From a42af19ca1efc1dbb678c333e09e607c9966354f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Thu, 29 Oct 2020 11:37:06 +0100 Subject: [PATCH 22/38] Proper JSONAPI compliant error responses --- README.md | 4 ++-- helpers.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4fc74f2..dd9a426 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,9 @@ Validate whether the Content-Type header contains the JSONAPI `content-type`-hea Validate whether the type specified in the JSONAPI data is equal to the expected type. Returns a 409 otherwise. -#### error(title, status = 400) +#### error(title, status=400, **kwargs) -Returns a JSONAPI compliant error response with the given status code (default: 400). +Returns a JSONAPI compliant error [Response object](https://flask.palletsprojects.com/en/1.1.x/api/#response-objects) with the given status code (default: 400). `kwargs` can be any other keys supported by [JSONAPI error objects](https://jsonapi.org/format/#error-objects). #### query(query) diff --git a/helpers.py b/helpers.py index 628ae4f..ced10c0 100644 --- a/helpers.py +++ b/helpers.py @@ -48,10 +48,16 @@ def rewrite_url_header(request): return request.args.get('HTTP_X_REWRITE_URL') -def error(msg, status=400): - """Returns a JSONAPI compliant error response with the given status code (400 by default).""" - response = jsonify({'message': msg}) - response.status_code = status +def error(msg, status=400, **kwargs): + """Returns a Response object containing a JSONAPI compliant error response + with the given status code (400 by default).""" + error_obj = kwargs + error_obj["detail"] = msg + error_obj["status"] = status + response = jsonify({ + "errors": [error_obj] + }) + response.status_code = error_obj["status"] return response From ca4e29764a4168bb36c908b0f6e78e720596915c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Thu, 29 Oct 2020 11:41:02 +0100 Subject: [PATCH 23/38] Linting --- web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web.py b/web.py index f500348..813c9b2 100644 --- a/web.py +++ b/web.py @@ -36,5 +36,5 @@ ## Start Application ## ####################### if __name__ == '__main__': - debug = True if (os.environ.get('MODE') == "development") else False + debug = os.environ.get('MODE') == "development" app.run(debug=debug, host='0.0.0.0', port=80) From 6a2f145bf2d7d1d49c26a043ef274204490616c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Thu, 29 Oct 2020 12:08:51 +0100 Subject: [PATCH 24/38] Implement SPARQL_TIMEOUT as documented --- helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helpers.py b/helpers.py index ced10c0..512f99a 100644 --- a/helpers.py +++ b/helpers.py @@ -79,6 +79,10 @@ def validate_resource_type(expected_type, data): sparqlQuery = SPARQLWrapper(os.environ.get('MU_SPARQL_ENDPOINT'), returnFormat=JSON) sparqlUpdate = SPARQLWrapper(os.environ.get('MU_SPARQL_UPDATEPOINT'), returnFormat=JSON) sparqlUpdate.method = 'POST' +if os.environ.get('MU_SPARQL_TIMEOUT'): + timeout = int(os.environ.get('MU_SPARQL_TIMEOUT')) + sparqlQuery.setTimeout(timeout) + sparqlUpdate.setTimeout(timeout) def query(the_query): """Execute the given SPARQL query (select/ask/construct)on the tripple store and returns the results From 3b3302a7842af54ebe2dded11def2ab55cfe3789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Thu, 29 Oct 2020 12:10:21 +0100 Subject: [PATCH 25/38] Comment regarding helper method --- helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers.py b/helpers.py index 512f99a..2da60ad 100644 --- a/helpers.py +++ b/helpers.py @@ -119,7 +119,7 @@ def update_modified(subject, modified=datetime.datetime.now()): query += " }" update(query) - +# TODO: Not sure what this function is for, since strings are supposed to be escaped. def verify_string_parameter(parameter): if parameter and type(parameter) is str: if "insert" in parameter.lower(): return error("unauthorized insert in string parameter") From 8d2c3b0290c2b93dcd7cbbc69967e215e7bf9781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Mon, 9 Nov 2020 11:08:01 +0100 Subject: [PATCH 26/38] Add template-specific linting-config --- README.md | 5 + template.pylintrc | 589 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 594 insertions(+) create mode 100644 template.pylintrc diff --git a/README.md b/README.md index dd9a426..c2f9870 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,11 @@ example docker-compose parameters: - /home/my/code/my-python-service:/app ``` +### Linting + +Since the template provides some dependencies that `pylint` doesn't know about while you're writing your services' code, pylint might unrightfully have some complaints. You can use the provided `template.pylintrc` as `.pylintrc`-file in your service. This configuration-file provides the necessary specifics for services based on the Python template. + + ### Helper methods The template provides the user with several helper methods. They aim to give you a step ahead for: diff --git a/template.pylintrc b/template.pylintrc new file mode 100644 index 0000000..fc90e82 --- /dev/null +++ b/template.pylintrc @@ -0,0 +1,589 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape, + missing-module-docstring, + relative-beyond-top-level + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules=helpers,escape_helpers,flask,SPARQLWrapper,rdflib,signinghub_api_client + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, `new` is for `{}` formatting,and `fstr` is for f-strings. +logging-format-style=new + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: en_ZW (myspell), en_NA +# (myspell), en_BS (myspell), en_US (myspell), en_NZ (myspell), en_JM +# (myspell), en_CA (myspell), en_IN (myspell), en_GH (myspell), en_AU +# (myspell), en_PH (myspell), en_IE (myspell), en_AG (myspell), en_BZ +# (myspell), en_ZA (myspell), en_ZM (myspell), en_DK (myspell), en_TT +# (myspell), en_BW (myspell), en_HK (myspell), en_NG (myspell), en_GB +# (myspell), en_MW (myspell), en_SG (myspell). +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether the implicit-str-concat-in-sequence should +# generate a warning on implicit string concatenation in sequences defined over +# several lines. +check-str-concat-over-line-jumps=no + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins=app,helpers,sparql_escape + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + f, + ex, + Run, + _ + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception From df5f46298edcc348c2b93ffd3713ba9c7584c302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Wed, 18 Nov 2020 12:28:08 +0100 Subject: [PATCH 27/38] Add correct content-type for JSONAPI error message --- helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/helpers.py b/helpers.py index 2da60ad..e4173e3 100644 --- a/helpers.py +++ b/helpers.py @@ -58,6 +58,7 @@ def error(msg, status=400, **kwargs): "errors": [error_obj] }) response.status_code = error_obj["status"] + response.headers["Content-Type"] = "application/vnd.api+json" return response From e79b740e654173624ec2daf6932b3f541f3cd104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 8 Jan 2021 11:41:39 +0100 Subject: [PATCH 28/38] fix: get headers from correct property --- helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers.py b/helpers.py index e4173e3..1f17571 100644 --- a/helpers.py +++ b/helpers.py @@ -40,12 +40,12 @@ def log(msg, *args, **kwargs): def session_id_header(request): """returns the HTTP_MU_SESSION_ID header from the given request""" - return request.args.get('HTTP_MU_SESSION_ID') + return request.headers.get('HTTP_MU_SESSION_ID') def rewrite_url_header(request): """return the HTTP_X_REWRTITE_URL header from the given request""" - return request.args.get('HTTP_X_REWRITE_URL') + return request.headers.get('HTTP_X_REWRITE_URL') def error(msg, status=400, **kwargs): From 95668440270da81e3933d812b65fcbb4be0ce2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 8 Jan 2021 11:43:12 +0100 Subject: [PATCH 29/38] feat: pass through mu headers for sparql queries --- helpers.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/helpers.py b/helpers.py index 1f17571..5f8c518 100644 --- a/helpers.py +++ b/helpers.py @@ -3,7 +3,7 @@ import logging import os import sys -from flask import jsonify +from flask import jsonify, request from rdflib.namespace import DC from escape_helpers import sparql_escape from SPARQLWrapper import SPARQLWrapper, JSON @@ -85,10 +85,19 @@ def validate_resource_type(expected_type, data): sparqlQuery.setTimeout(timeout) sparqlUpdate.setTimeout(timeout) +MU_HEADERS = [ + "HTTP_MU_CALL_ID", + "HTTP_MU_AUTH_ALLOWED_GROUPS", + "HTTP_MU_AUTH_USED_GROUPS" +] + def query(the_query): """Execute the given SPARQL query (select/ask/construct)on the tripple store and returns the results in the given returnFormat (JSON by default).""" log("execute query: \n" + the_query) + sparqlQuery.customHttpHeaders["HTTP_MU_SESSION_ID"] = session_id_header(request) + for header in MU_HEADERS: + sparqlQuery.customHttpHeaders[header] = request.headers.get(header) sparqlQuery.setQuery(the_query) return sparqlQuery.query().convert() @@ -96,6 +105,9 @@ def query(the_query): def update(the_query): """Execute the given update SPARQL query on the tripple store, if the given query is no update query, nothing happens.""" + sparqlQuery.customHttpHeaders["HTTP_MU_SESSION_ID"] = session_id_header(request) + for header in MU_HEADERS: + sparqlQuery.customHttpHeaders[header] = request.headers.get(header) sparqlUpdate.setQuery(the_query) if sparqlUpdate.isSparqlUpdateRequest(): sparqlUpdate.query() From d1a32dc02e455bf3395e840a3ea7582ecd0ab283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 8 Jan 2021 12:20:20 +0100 Subject: [PATCH 30/38] fix: Make sure headers cannot be set to None --- helpers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/helpers.py b/helpers.py index 5f8c518..cd52849 100644 --- a/helpers.py +++ b/helpers.py @@ -97,7 +97,11 @@ def query(the_query): log("execute query: \n" + the_query) sparqlQuery.customHttpHeaders["HTTP_MU_SESSION_ID"] = session_id_header(request) for header in MU_HEADERS: - sparqlQuery.customHttpHeaders[header] = request.headers.get(header) + if header in request.headers: + sparqlQuery.customHttpHeaders[header] = request.headers[header] + else: # Make sure headers used for a previous query are cleared + if header in request.customHttpHeaders: + del sparqlQuery.customHttpHeaders[header] sparqlQuery.setQuery(the_query) return sparqlQuery.query().convert() @@ -107,7 +111,11 @@ def update(the_query): if the given query is no update query, nothing happens.""" sparqlQuery.customHttpHeaders["HTTP_MU_SESSION_ID"] = session_id_header(request) for header in MU_HEADERS: - sparqlQuery.customHttpHeaders[header] = request.headers.get(header) + if header in request.headers: + sparqlQuery.customHttpHeaders[header] = request.headers[header] + else: # Make sure headers used for a previous query are cleared + if header in request.customHttpHeaders: + del sparqlQuery.customHttpHeaders[header] sparqlUpdate.setQuery(the_query) if sparqlUpdate.isSparqlUpdateRequest(): sparqlUpdate.query() From 257c677df66f1c96dd10f28f61cbe04578ea5719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 8 Jan 2021 12:27:00 +0100 Subject: [PATCH 31/38] fix: correct sparql header dict name --- helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers.py b/helpers.py index cd52849..23c16d3 100644 --- a/helpers.py +++ b/helpers.py @@ -100,7 +100,7 @@ def query(the_query): if header in request.headers: sparqlQuery.customHttpHeaders[header] = request.headers[header] else: # Make sure headers used for a previous query are cleared - if header in request.customHttpHeaders: + if header in sparqlQuery.customHttpHeaders: del sparqlQuery.customHttpHeaders[header] sparqlQuery.setQuery(the_query) return sparqlQuery.query().convert() @@ -114,7 +114,7 @@ def update(the_query): if header in request.headers: sparqlQuery.customHttpHeaders[header] = request.headers[header] else: # Make sure headers used for a previous query are cleared - if header in request.customHttpHeaders: + if header in sparqlQuery.customHttpHeaders: del sparqlQuery.customHttpHeaders[header] sparqlUpdate.setQuery(the_query) if sparqlUpdate.isSparqlUpdateRequest(): From 56e6aba2ceac0fa8ca57f7ffdf78fa29af9f3061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 8 Jan 2021 12:48:34 +0100 Subject: [PATCH 32/38] fix: take session id header into loop to avoid conditionals duplication --- helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/helpers.py b/helpers.py index 23c16d3..7de6e83 100644 --- a/helpers.py +++ b/helpers.py @@ -86,6 +86,7 @@ def validate_resource_type(expected_type, data): sparqlUpdate.setTimeout(timeout) MU_HEADERS = [ + "HTTP_MU_SESSION_ID", "HTTP_MU_CALL_ID", "HTTP_MU_AUTH_ALLOWED_GROUPS", "HTTP_MU_AUTH_USED_GROUPS" @@ -95,7 +96,6 @@ def query(the_query): """Execute the given SPARQL query (select/ask/construct)on the tripple store and returns the results in the given returnFormat (JSON by default).""" log("execute query: \n" + the_query) - sparqlQuery.customHttpHeaders["HTTP_MU_SESSION_ID"] = session_id_header(request) for header in MU_HEADERS: if header in request.headers: sparqlQuery.customHttpHeaders[header] = request.headers[header] @@ -109,7 +109,6 @@ def query(the_query): def update(the_query): """Execute the given update SPARQL query on the tripple store, if the given query is no update query, nothing happens.""" - sparqlQuery.customHttpHeaders["HTTP_MU_SESSION_ID"] = session_id_header(request) for header in MU_HEADERS: if header in request.headers: sparqlQuery.customHttpHeaders[header] = request.headers[header] From ca7fdc266de5908eadd4958d4ab4545a907d56e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Fri, 8 Jan 2021 13:47:17 +0100 Subject: [PATCH 33/38] fix: ruby rack style http header notation --- helpers.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/helpers.py b/helpers.py index 7de6e83..2d497d8 100644 --- a/helpers.py +++ b/helpers.py @@ -39,13 +39,13 @@ def log(msg, *args, **kwargs): def session_id_header(request): - """returns the HTTP_MU_SESSION_ID header from the given request""" - return request.headers.get('HTTP_MU_SESSION_ID') + """returns the MU-SESSION-ID header from the given request""" + return request.headers.get('MU-SESSION-ID') def rewrite_url_header(request): - """return the HTTP_X_REWRTITE_URL header from the given request""" - return request.headers.get('HTTP_X_REWRITE_URL') + """return the X-REWRITE-URL header from the given request""" + return request.headers.get('X-REWRITE-URL') def error(msg, status=400, **kwargs): @@ -86,10 +86,10 @@ def validate_resource_type(expected_type, data): sparqlUpdate.setTimeout(timeout) MU_HEADERS = [ - "HTTP_MU_SESSION_ID", - "HTTP_MU_CALL_ID", - "HTTP_MU_AUTH_ALLOWED_GROUPS", - "HTTP_MU_AUTH_USED_GROUPS" + "MU-SESSION-ID", + "MU-CALL-ID", + "MU-AUTH-ALLOWED-GROUPS", + "MU-AUTH-USED-GROUPS" ] def query(the_query): From 61ac53030968928b0e60bfbbf3891c6a5de5b82b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Mon, 11 Jan 2021 10:50:21 +0100 Subject: [PATCH 34/38] fix: modify update headers in update query, not query headers --- helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helpers.py b/helpers.py index 2d497d8..845f20e 100644 --- a/helpers.py +++ b/helpers.py @@ -111,10 +111,10 @@ def update(the_query): if the given query is no update query, nothing happens.""" for header in MU_HEADERS: if header in request.headers: - sparqlQuery.customHttpHeaders[header] = request.headers[header] + sparqlUpdate.customHttpHeaders[header] = request.headers[header] else: # Make sure headers used for a previous query are cleared - if header in sparqlQuery.customHttpHeaders: - del sparqlQuery.customHttpHeaders[header] + if header in sparqlUpdate.customHttpHeaders: + del sparqlUpdate.customHttpHeaders[header] sparqlUpdate.setQuery(the_query) if sparqlUpdate.isSparqlUpdateRequest(): sparqlUpdate.query() From b6c7ee169d10bb5d247d5abae9a613f325386d90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Mon, 28 Jun 2021 14:41:30 +0200 Subject: [PATCH 35/38] doc: remove references to linting config --- README.md | 5 - template.pylintrc | 589 ---------------------------------------------- 2 files changed, 594 deletions(-) delete mode 100644 template.pylintrc diff --git a/README.md b/README.md index c2f9870..dd9a426 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,6 @@ example docker-compose parameters: - /home/my/code/my-python-service:/app ``` -### Linting - -Since the template provides some dependencies that `pylint` doesn't know about while you're writing your services' code, pylint might unrightfully have some complaints. You can use the provided `template.pylintrc` as `.pylintrc`-file in your service. This configuration-file provides the necessary specifics for services based on the Python template. - - ### Helper methods The template provides the user with several helper methods. They aim to give you a step ahead for: diff --git a/template.pylintrc b/template.pylintrc deleted file mode 100644 index fc90e82..0000000 --- a/template.pylintrc +++ /dev/null @@ -1,589 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. -jobs=1 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# List of plugins (as comma separated values of python module names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, - missing-module-docstring, - relative-beyond-top-level - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a score less than or equal to 10. You -# have access to the variables 'error', 'warning', 'refactor', and 'convention' -# which contain the number of messages in each category, as well as 'statement' -# which is the total number of statements analyzed. This score is used by the -# global evaluation report (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis). It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules=helpers,escape_helpers,flask,SPARQLWrapper,rdflib,signinghub_api_client - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - -# List of decorators that change the signature of a decorated function. -signature-mutators= - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module. -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[LOGGING] - -# Format style used to check logging format string. `old` means using % -# formatting, `new` is for `{}` formatting,and `fstr` is for f-strings. -logging-format-style=new - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: en_ZW (myspell), en_NA -# (myspell), en_BS (myspell), en_US (myspell), en_NZ (myspell), en_JM -# (myspell), en_CA (myspell), en_IN (myspell), en_GH (myspell), en_AU -# (myspell), en_PH (myspell), en_IE (myspell), en_AG (myspell), en_BZ -# (myspell), en_ZA (myspell), en_ZM (myspell), en_DK (myspell), en_TT -# (myspell), en_BW (myspell), en_HK (myspell), en_NG (myspell), en_GB -# (myspell), en_MW (myspell), en_SG (myspell). -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains the private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -spelling-store-unknown-words=no - - -[STRING] - -# This flag controls whether the implicit-str-concat-in-sequence should -# generate a warning on implicit string concatenation in sequences defined over -# several lines. -check-str-concat-over-line-jumps=no - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins=app,helpers,sparql_escape - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[BASIC] - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style. -#argument-rgx= - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma. -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. -#class-attribute-rgx= - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming- -# style. -#class-rgx= - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style. -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style. -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma. -good-names=i, - j, - k, - f, - ex, - Run, - _ - -# Include a hint for the correct naming format with invalid-name. -include-naming-hint=no - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. -#inlinevar-rgx= - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style. -#method-rgx= - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style. -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -# These decorators are taken in consideration only for invalid-name. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style. -#variable-rgx= - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[DESIGN] - -# Maximum number of arguments for function / method. -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in an if statement (see R0916). -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=12 - -# Maximum number of locals for function / method body. -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body. -max-returns=6 - -# Maximum number of statements in function / method body. -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp, - __post_init__ - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls - - -[IMPORTS] - -# List of modules that can be imported at any level, not just the top level -# one. -allow-any-import-level= - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Couples of modules and preferred modules, separated by a comma. -preferred-modules= - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception From 48dff8b9ba741d11f3b20379c196b66601d7ebbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Mon, 28 Jun 2021 14:53:54 +0200 Subject: [PATCH 36/38] fix: don't force timestamps to second-precision. allow ms & us --- escape_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/escape_helpers.py b/escape_helpers.py index 9af99d3..2129ea6 100644 --- a/escape_helpers.py +++ b/escape_helpers.py @@ -13,8 +13,8 @@ def sparql_escape_datetime(obj): if not isinstance(obj, datetime.datetime): warn("You are escaping something that isn't a datetime with \ the 'sparql_escape_datetime'-method. Implicit casting will occurr.") - obj = datetime.datetime.fromisoformat(str(obj)) - return '"{}"^^xsd:dateTime'.format(obj.isoformat(timespec="seconds")) + obj = datetime.datetime.fromisoformat(str(obj)) # only supports 3 or 6 digit microsecond notation (https://docs.python.org/3.7/library/datetime.html#datetime.datetime.fromisoformat) + return '"{}"^^xsd:dateTime'.format(obj.isoformat()) def sparql_escape_date(obj): if not isinstance(obj, datetime.date): @@ -27,8 +27,8 @@ def sparql_escape_time(obj): if not isinstance(obj, datetime.time): warn("You are escaping something that isn't a time with \ the 'sparql_escape_time'-method. Implicit casting will occurr.") - obj = datetime.time.fromisoformat(str(obj)) - return '"{}"^^xsd:time'.format(obj.isoformat(timespec="seconds")) + obj = datetime.time.fromisoformat(str(obj)) # only supports 3 or 6 digit microsecond notation (https://docs.python.org/3.7/library/datetime.html#datetime.time.fromisoformat) + return '"{}"^^xsd:time'.format(obj.isoformat()) def sparql_escape_int(obj): if not isinstance(obj, int): From c95699f30db3676611db4066b70680f83feac06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Mon, 28 Jun 2021 15:04:46 +0200 Subject: [PATCH 37/38] feat: configure amount of workers to be 1 by default + doc --- Dockerfile | 1 + README.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index f008ec7..2f53b5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ MAINTAINER Michaël Dierick "michael@dierick.io" # Gunicorn Docker config ENV MODULE_NAME web ENV PYTHONPATH "/usr/src/app" +ENV WEB_CONCURRENCY "1" # Overrides the start.sh used in `tiangolo/meinheld-gunicorn` COPY ./start.sh /start.sh diff --git a/README.md b/README.md index dd9a426..bb78c90 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,8 @@ my-python: - `MU_SPARQL_TIMEOUT` is used to configure the timeout (in seconds) for SPARQL queries. +Since this template is based on the meinheld-gunicorn-docker image, all possible environment config for that image is also available for the template. See [meinheld-gunicorn-docker#environment-variables](https://github.com/tiangolo/meinheld-gunicorn-docker#environment-variables) for more info. The template configures `WEB_CONCURRENCY` in particular to `1` by default. + ### Production For hosting the app in a production setting, the template depends on [meinheld-gunicorn-docker](https://github.com/tiangolo/meinheld-gunicorn-docker). All [environment variables](https://github.com/tiangolo/meinheld-gunicorn-docker#environment-variables) used by meinheld-gunicorn can be used to configure your service as well. From bf3687465d74653d4438957e255c3cdc2ec4ec69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Dierick?= Date: Tue, 6 Jul 2021 14:57:32 +0200 Subject: [PATCH 38/38] remove unused `verify_string_parameter` helper function. --- helpers.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/helpers.py b/helpers.py index 845f20e..ce6f1fc 100644 --- a/helpers.py +++ b/helpers.py @@ -138,16 +138,3 @@ def update_modified(subject, modified=datetime.datetime.now()): query += " }" query += " }" update(query) - -# TODO: Not sure what this function is for, since strings are supposed to be escaped. -def verify_string_parameter(parameter): - if parameter and type(parameter) is str: - if "insert" in parameter.lower(): return error("unauthorized insert in string parameter") - if "delete" in parameter.lower(): return error("unauthorized delete in string parameter") - if "load" in parameter.lower(): return error("unauthorized load in string parameter") - if "clear" in parameter.lower(): return error("unauthorized clear in string parameter") - if "create" in parameter.lower(): return error("unauthorized create in string parameter") - if "drop" in parameter.lower(): return error("unauthorized drop in string parameter") - if "copy" in parameter.lower(): return error("unauthorized copy in string parameter") - if "move" in parameter.lower(): return error("unauthorized move in string parameter") - if "add" in parameter.lower(): return error("unauthorized add in string parameter")