Detect source code encoding in .ptl files.

quixote/ptl/ptl_compile.py

@@ -16,6 +16,7 @@ import re
 import imp
 import marshal
 import struct
+import importlib.util
 
 HTML_TEMPLATE_PREFIX = "_q_html_template_"
 PLAIN_TEMPLATE_PREFIX = "_q_plain_template_"
@@ -124,12 +125,12 @@ class TemplateTransformer(ast.NodeTransformer):
         else:
             return node
 
-_template_re = re.compile(
-    r"^(?P<indent>[ \t]*) def (?:[ \t]+)"
-    r" (?P<name>[a-zA-Z_][a-zA-Z_0-9]*)"
-    r" (?:[ \t]*) \[(?P<type>plain|html)\] (?:[ \t]*)"
-    r" (?:[ \t]*[\(\\])",
-    re.MULTILINE|re.VERBOSE)
+_template_re = re.compile(r'''
+    ^(?P<indent>[ \t]*) def (?:[ \t]+)
+    (?P<name>[a-zA-Z_][a-zA-Z_0-9]*)
+    (?:[ \t]*) \[(?P<type>plain|html)\] (?:[ \t]*)
+    (?:[ \t]*[\(\\])
+    ''', re.MULTILINE|re.VERBOSE)
 
 def translate_tokens(buf):
     """
@@ -152,6 +153,8 @@ def translate_tokens(buf):
     return  _template_re.sub(replacement, buf)
 
 def parse(buf, filename='<string>'):
+    if isinstance(buf, bytes):
+        buf = importlib.util.decode_source(buf)
     buf = translate_tokens(buf)
     try:
         node = ast.parse(buf, filename)

quixote/ptl/ptl_import.py

@@ -10,9 +10,6 @@ from quixote.ptl.ptl_compile import parse, PTL_EXT
 class PTLFileLoader(SourceFileLoader):
     @staticmethod
     def source_to_code(data, path='<string>'):
-        if isinstance(data, bytes):
-            # FIXME: we should check the encoding of the source file
-            data = data.decode('utf-8')
         node = parse(data, path)
         return compile(node, path, 'exec')