diff --git a/quixote/ptl/ptl_compile.py b/quixote/ptl/ptl_compile.py index 533d572..c8385ab 100644 --- a/quixote/ptl/ptl_compile.py +++ b/quixote/ptl/ptl_compile.py @@ -16,6 +16,7 @@ import re import imp import marshal import struct +import importlib.util HTML_TEMPLATE_PREFIX = "_q_html_template_" PLAIN_TEMPLATE_PREFIX = "_q_plain_template_" @@ -124,12 +125,12 @@ class TemplateTransformer(ast.NodeTransformer): else: return node -_template_re = re.compile( - r"^(?P[ \t]*) def (?:[ \t]+)" - r" (?P[a-zA-Z_][a-zA-Z_0-9]*)" - r" (?:[ \t]*) \[(?Pplain|html)\] (?:[ \t]*)" - r" (?:[ \t]*[\(\\])", - re.MULTILINE|re.VERBOSE) +_template_re = re.compile(r''' + ^(?P[ \t]*) def (?:[ \t]+) + (?P[a-zA-Z_][a-zA-Z_0-9]*) + (?:[ \t]*) \[(?Pplain|html)\] (?:[ \t]*) + (?:[ \t]*[\(\\]) + ''', re.MULTILINE|re.VERBOSE) def translate_tokens(buf): """ @@ -152,6 +153,8 @@ def translate_tokens(buf): return _template_re.sub(replacement, buf) def parse(buf, filename=''): + if isinstance(buf, bytes): + buf = importlib.util.decode_source(buf) buf = translate_tokens(buf) try: node = ast.parse(buf, filename) diff --git a/quixote/ptl/ptl_import.py b/quixote/ptl/ptl_import.py index 90d1947..0ce58c2 100644 --- a/quixote/ptl/ptl_import.py +++ b/quixote/ptl/ptl_import.py @@ -10,9 +10,6 @@ from quixote.ptl.ptl_compile import parse, PTL_EXT class PTLFileLoader(SourceFileLoader): @staticmethod def source_to_code(data, path=''): - if isinstance(data, bytes): - # FIXME: we should check the encoding of the source file - data = data.decode('utf-8') node = parse(data, path) return compile(node, path, 'exec')