Python 3 changes: use ast module to compile, importlib hooks.
This commit is contained in:
Neil Schemenauer
parent
2b7b5a8f0e
commit
31dceaf37d
|
|
@ -5,37 +5,27 @@ First template function names are mangled, noting the template type.
|
|||
Next, the file is parsed into a parse tree. This tree is converted into
|
||||
a modified AST. It is during this state that the semantics are modified
|
||||
by adding extra nodes to the tree. Finally bytecode is generated using
|
||||
the compiler package.
|
||||
the compiler.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import sys
|
||||
import os
|
||||
import stat
|
||||
import symbol
|
||||
import token
|
||||
import re
|
||||
import imp
|
||||
import stat
|
||||
import marshal
|
||||
import struct
|
||||
|
||||
assert sys.hexversion >= 0x20300b1, 'PTL requires Python 2.3 or newer'
|
||||
|
||||
from compiler import pycodegen, transformer
|
||||
from compiler import ast
|
||||
from compiler.consts import OP_ASSIGN
|
||||
from compiler import misc, syntax
|
||||
|
||||
HTML_TEMPLATE_PREFIX = "_q_html_template_"
|
||||
PLAIN_TEMPLATE_PREFIX = "_q_plain_template_"
|
||||
|
||||
class TemplateTransformer(transformer.Transformer):
|
||||
|
||||
class TemplateTransformer(ast.NodeTransformer):
|
||||
def __init__(self, *args, **kwargs):
|
||||
transformer.Transformer.__init__(self, *args, **kwargs)
|
||||
ast.NodeTransformer.__init__(self, *args, **kwargs)
|
||||
# __template_type is a stack whose values are
|
||||
# "html", "plain", or None
|
||||
self.__template_type = []
|
||||
self.__template_type = [None]
|
||||
|
||||
def _get_template_type(self):
|
||||
"""Return the type of the function being compiled (
|
||||
|
|
@ -46,129 +36,93 @@ class TemplateTransformer(transformer.Transformer):
|
|||
else:
|
||||
return None
|
||||
|
||||
def file_input(self, nodelist):
|
||||
doc = None # self.get_docstring(nodelist, symbol.file_input)
|
||||
if sys.hexversion >= 0x02050000:
|
||||
html_imp = ast.From(
|
||||
'quixote.html',
|
||||
[('TemplateIO', '_q_TemplateIO'), ('htmltext', '_q_htmltext')],
|
||||
0)
|
||||
vars_imp = ast.From("__builtin__", [("vars", "_q_vars")], 0)
|
||||
else:
|
||||
html_imp = ast.From(
|
||||
'quixote.html',
|
||||
[('TemplateIO', '_q_TemplateIO'), ('htmltext', '_q_htmltext')])
|
||||
vars_imp = ast.From("__builtin__", [("vars", "_q_vars")])
|
||||
|
||||
ptl_imports = [ vars_imp, html_imp ]
|
||||
stmts = []
|
||||
for node in nodelist:
|
||||
if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
|
||||
self.com_append_stmt(stmts, node)
|
||||
def visit_Module(self, node):
|
||||
html_imp = ast.ImportFrom(module='quixote.html',
|
||||
names=[ast.alias(name='TemplateIO',
|
||||
asname='_q_TemplateIO'),
|
||||
ast.alias(name='htmltext',
|
||||
asname='_q_htmltext')],
|
||||
level=0)
|
||||
ast.fix_missing_locations(html_imp)
|
||||
vars_imp = ast.ImportFrom(module='builtins',
|
||||
names=[ast.alias(name='vars',
|
||||
asname='_q_vars')], level=0)
|
||||
ast.fix_missing_locations(vars_imp)
|
||||
ptl_imports = [vars_imp, html_imp]
|
||||
# count __future__ statements
|
||||
i = 0
|
||||
for stmt in stmts:
|
||||
if isinstance(stmt, ast.From) and stmt.modname == '__future__':
|
||||
for stmt in node.body:
|
||||
if isinstance(stmt, ast.ImportFrom) and stmt.module == '__future__':
|
||||
i += 1
|
||||
else:
|
||||
break
|
||||
stmts[i:i] = ptl_imports
|
||||
return ast.Module(doc, ast.Stmt(stmts))
|
||||
node.body[i:i] = ptl_imports
|
||||
return self.generic_visit(node)
|
||||
|
||||
def funcdef(self, nodelist):
|
||||
if len(nodelist) == 6:
|
||||
assert nodelist[0][0] == symbol.decorators
|
||||
decorators = self.decorators(nodelist[0][1:])
|
||||
else:
|
||||
assert len(nodelist) == 5
|
||||
decorators = None
|
||||
|
||||
lineno = nodelist[-4][2]
|
||||
name = nodelist[-4][1]
|
||||
args = nodelist[-3][2]
|
||||
|
||||
if not re.match('_q_(html|plain)_(dollar_)?template_', name):
|
||||
# just a normal function, let base class handle it
|
||||
def visit_FunctionDef(self, node):
|
||||
name = node.name
|
||||
if not re.match('_q_(html|plain)_template_', name):
|
||||
# just a normal function
|
||||
self.__template_type.append(None)
|
||||
n = transformer.Transformer.funcdef(self, nodelist)
|
||||
node = self.generic_visit(node)
|
||||
else:
|
||||
if name.startswith(PLAIN_TEMPLATE_PREFIX):
|
||||
name = name[len(PLAIN_TEMPLATE_PREFIX):]
|
||||
node.name = name[len(PLAIN_TEMPLATE_PREFIX):]
|
||||
template_type = "plain"
|
||||
elif name.startswith(HTML_TEMPLATE_PREFIX):
|
||||
name = name[len(HTML_TEMPLATE_PREFIX):]
|
||||
node.name = name[len(HTML_TEMPLATE_PREFIX):]
|
||||
template_type = "html"
|
||||
else:
|
||||
raise RuntimeError, 'unknown prefix on %s' % name
|
||||
raise RuntimeError('unknown prefix on %s' % name)
|
||||
|
||||
self.__template_type.append(template_type)
|
||||
|
||||
if args[0] == symbol.varargslist:
|
||||
names, defaults, flags = self.com_arglist(args[1:])
|
||||
else:
|
||||
names = defaults = ()
|
||||
flags = 0
|
||||
doc = None # self.get_docstring(nodelist[-1])
|
||||
|
||||
# code for function
|
||||
code = self.com_node(nodelist[-1])
|
||||
node = self.generic_visit(node)
|
||||
|
||||
# _q_output = _q_TemplateIO()
|
||||
klass = ast.Name('_q_TemplateIO')
|
||||
args = [ast.Const(template_type == "html")]
|
||||
instance = ast.CallFunc(klass, args)
|
||||
assign_name = ast.AssName('_q_output', OP_ASSIGN)
|
||||
assign = ast.Assign([assign_name], instance)
|
||||
klass = ast.Name(id='_q_TemplateIO', ctx=ast.Load())
|
||||
arg = ast.NameConstant(template_type == 'html')
|
||||
instance = ast.Call(func=klass, args=[arg], keywords=[],
|
||||
starargs=None, kwargs=None)
|
||||
assign_name = ast.Name(id='_q_output', ctx=ast.Store())
|
||||
assign = ast.Assign(targets=[assign_name], value=instance)
|
||||
ast.copy_location(assign, node)
|
||||
ast.fix_missing_locations(assign)
|
||||
node.body.insert(0, assign)
|
||||
|
||||
# return _q_output.getvalue()
|
||||
func = ast.Getattr(ast.Name('_q_output'), "getvalue")
|
||||
ret = ast.Return(ast.CallFunc(func, []))
|
||||
|
||||
# wrap original function code
|
||||
code = ast.Stmt([assign, code, ret])
|
||||
code.lineno = lineno
|
||||
|
||||
if sys.hexversion >= 0x20400a2:
|
||||
n = ast.Function(decorators, name, names, defaults, flags, doc,
|
||||
code)
|
||||
else:
|
||||
n = ast.Function(name, names, defaults, flags, doc, code)
|
||||
n.lineno = lineno
|
||||
n = ast.Name(id='_q_output', ctx=ast.Load())
|
||||
n = ast.Attribute(value=n, attr='getvalue', ctx=ast.Load())
|
||||
n = ast.Call(func=n, args=[], keywords=[], starargs=None,
|
||||
kwargs=None)
|
||||
ret = ast.Return(value=n)
|
||||
ast.copy_location(ret, node.body[-1])
|
||||
ast.fix_missing_locations(ret)
|
||||
node.body.append(ret)
|
||||
|
||||
self.__template_type.pop()
|
||||
return n
|
||||
return node
|
||||
|
||||
def expr_stmt(self, nodelist):
|
||||
if self._get_template_type() is None:
|
||||
return transformer.Transformer.expr_stmt(self, nodelist)
|
||||
|
||||
# Instead of discarding objects on the stack, call
|
||||
# "_q_output += obj".
|
||||
exprNode = self.com_node(nodelist[-1])
|
||||
if len(nodelist) == 1:
|
||||
lval = ast.Name('_q_output')
|
||||
n = ast.AugAssign(lval, '+=', exprNode)
|
||||
if hasattr(exprNode, 'lineno'):
|
||||
n.lineno = exprNode.lineno
|
||||
elif nodelist[1][0] == token.EQUAL:
|
||||
nodes = [ ]
|
||||
for i in range(0, len(nodelist) - 2, 2):
|
||||
nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
|
||||
n = ast.Assign(nodes, exprNode)
|
||||
n.lineno = nodelist[1][2]
|
||||
def visit_Expr(self, node):
|
||||
if self._get_template_type() is not None:
|
||||
node = self.generic_visit(node)
|
||||
# Instead of discarding objects on the stack, call
|
||||
# "_q_output += obj".
|
||||
lval = ast.Name(id='_q_output', ctx=ast.Store())
|
||||
ast.copy_location(lval, node)
|
||||
aug = ast.AugAssign(target=lval, op=ast.Add(), value=node.value)
|
||||
return ast.copy_location(aug, node)
|
||||
else:
|
||||
lval = self.com_augassign(nodelist[0])
|
||||
op = self.com_augassign_op(nodelist[1])
|
||||
n = ast.AugAssign(lval, op[1], exprNode)
|
||||
n.lineno = op[2]
|
||||
return n
|
||||
return node
|
||||
|
||||
def atom_string(self, nodelist):
|
||||
const_node = transformer.Transformer.atom_string(self, nodelist)
|
||||
def visit_Str(self, node):
|
||||
if "html" == self._get_template_type():
|
||||
return ast.CallFunc(ast.Name('_q_htmltext'), [const_node])
|
||||
n = ast.Name(id='_q_htmltext', ctx=ast.Load())
|
||||
ast.copy_location(n, node)
|
||||
n = ast.Call(func=n, args=[node], keywords=[], starargs=None,
|
||||
kwargs=None)
|
||||
return ast.copy_location(n, node)
|
||||
else:
|
||||
return const_node
|
||||
return node
|
||||
|
||||
_template_re = re.compile(
|
||||
r"^(?P<indent>[ \t]*) def (?:[ \t]+)"
|
||||
|
|
@ -200,31 +154,26 @@ def translate_tokens(buf):
|
|||
def parse(buf, filename='<string>'):
|
||||
buf = translate_tokens(buf)
|
||||
try:
|
||||
return TemplateTransformer().parsesuite(buf)
|
||||
except SyntaxError, e:
|
||||
node = ast.parse(buf, filename)
|
||||
except SyntaxError as e:
|
||||
# set the filename attribute
|
||||
raise SyntaxError(str(e), (filename, e.lineno, e.offset, e.text))
|
||||
t = TemplateTransformer()
|
||||
return t.visit(node)
|
||||
|
||||
|
||||
PTL_EXT = ".ptl"
|
||||
|
||||
class Template(pycodegen.Module):
|
||||
|
||||
def _get_tree(self):
|
||||
tree = parse(self.source, self.filename)
|
||||
misc.set_filename(self.filename, tree)
|
||||
syntax.check(tree)
|
||||
return tree
|
||||
|
||||
def dump(self, fp):
|
||||
mtime = os.stat(self.filename)[stat.ST_MTIME]
|
||||
fp.write('\0\0\0\0')
|
||||
fp.write(struct.pack('<I', mtime))
|
||||
marshal.dump(self.code, fp)
|
||||
fp.flush()
|
||||
fp.seek(0)
|
||||
fp.write(imp.get_magic())
|
||||
def dump(code, filename, fp):
|
||||
mtime = os.stat(filename)[stat.ST_MTIME]
|
||||
fp.write('\0\0\0\0')
|
||||
fp.write(struct.pack('<I', mtime))
|
||||
marshal.dump(code, fp)
|
||||
fp.flush()
|
||||
fp.seek(0)
|
||||
fp.write(imp.get_magic())
|
||||
|
||||
_compile = compile
|
||||
|
||||
def compile_template(input, filename, output=None):
|
||||
"""(input, filename, output=None) -> code
|
||||
|
|
@ -233,12 +182,11 @@ def compile_template(input, filename, output=None):
|
|||
If output is not None then the code is written to output.
|
||||
The code object is returned.
|
||||
"""
|
||||
buf = input.read()
|
||||
template = Template(buf, filename)
|
||||
template.compile()
|
||||
node = parse(input.read(), filename)
|
||||
code = _compile(node, filename, 'exec')
|
||||
if output is not None:
|
||||
template.dump(output)
|
||||
return template.code
|
||||
dump(code, filename, output)
|
||||
return code
|
||||
|
||||
def compile(inputname, outputname):
|
||||
"""(inputname, outputname)
|
||||
|
|
@ -266,7 +214,7 @@ def compile_file(filename, force=0, verbose=0):
|
|||
if (ctime > ftime) and not force:
|
||||
return
|
||||
if verbose:
|
||||
print 'Compiling', filename, '...'
|
||||
print('Compiling', filename, '...')
|
||||
ok = compile(filename, cfile)
|
||||
|
||||
def compile_dir(dir, maxlevels=10, force=0):
|
||||
|
|
@ -279,11 +227,11 @@ def compile_dir(dir, maxlevels=10, force=0):
|
|||
maxlevels: maximum recursion level (default 10)
|
||||
force: if true, force compilation, even if timestamps are up-to-date
|
||||
"""
|
||||
print 'Listing', dir, '...'
|
||||
print('Listing', dir, '...')
|
||||
try:
|
||||
names = os.listdir(dir)
|
||||
except os.error:
|
||||
print "Can't list", dir
|
||||
print("Can't list", dir)
|
||||
names = []
|
||||
names.sort()
|
||||
success = 1
|
||||
|
|
@ -299,8 +247,8 @@ def compile_dir(dir, maxlevels=10, force=0):
|
|||
if type(sys.exc_type) == type(''):
|
||||
exc_type_name = sys.exc_type
|
||||
else: exc_type_name = sys.exc_type.__name__
|
||||
print 'Sorry:', exc_type_name + ':',
|
||||
print sys.exc_value
|
||||
print('Sorry:', exc_type_name + ':',)
|
||||
print(sys.exc_value)
|
||||
success = 0
|
||||
else:
|
||||
if ok == 0:
|
||||
|
|
@ -327,7 +275,7 @@ def compile_package(path, force=0, verbose=0):
|
|||
def main():
|
||||
args = sys.argv[1:]
|
||||
if not args:
|
||||
print "no files to compile"
|
||||
print("no files to compile")
|
||||
else:
|
||||
for filename in args:
|
||||
path, ext = os.path.splitext(filename)
|
||||
|
|
|
|||
|
|
@ -1,156 +1,44 @@
|
|||
"""Import hooks; when installed, these hooks allow importing .ptl files
|
||||
as if they were Python modules.
|
||||
|
||||
Note: there's some unpleasant incompatibility between ZODB's import
|
||||
trickery and the import hooks here. Bottom line: if you're using ZODB,
|
||||
import it *before* installing the PTL import hooks.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os.path
|
||||
import imp, ihooks, new
|
||||
import struct
|
||||
import marshal
|
||||
import __builtin__
|
||||
from importlib.machinery import FileFinder, PathFinder, SourceFileLoader
|
||||
|
||||
# Check for a deficient ihooks module. Python 2.6 was released without
|
||||
# ihooks.py being updated to support relative imports. Any library that uses
|
||||
# relative imports will cause the import hook to fail. Use our local copy of
|
||||
# ihooks module which does have support for relative imports.
|
||||
if sys.hexversion >= 0x20600b0:
|
||||
_m = ihooks.ModuleImporter.import_module
|
||||
if _m.im_func.func_code.co_argcount == 5:
|
||||
import ihooks_local as ihooks
|
||||
from quixote.ptl.ptl_compile import parse, PTL_EXT
|
||||
|
||||
from quixote.ptl.ptl_compile import compile_template, PTL_EXT
|
||||
class PTLFileLoader(SourceFileLoader):
|
||||
@staticmethod
|
||||
def source_to_code(data, path='<string>'):
|
||||
if isinstance(data, bytes):
|
||||
# FIXME: we should check the encoding of the source file
|
||||
data = data.decode('utf-8')
|
||||
node = parse(data, path)
|
||||
return compile(node, path, 'exec')
|
||||
|
||||
assert sys.hexversion >= 0x20000b1, "need Python 2.0b1 or later"
|
||||
|
||||
def _exec_module_code(code, name, filename):
|
||||
if name in sys.modules:
|
||||
mod = sys.modules[name] # necessary for reload()
|
||||
else:
|
||||
mod = new.module(name)
|
||||
sys.modules[name] = mod
|
||||
mod.__name__ = name
|
||||
mod.__file__ = filename
|
||||
exec code in mod.__dict__
|
||||
return mod
|
||||
class PTLPathFinder(PathFinder):
|
||||
path_importer_cache = {}
|
||||
|
||||
def _timestamp(filename):
|
||||
try:
|
||||
s = os.stat(filename)
|
||||
except OSError:
|
||||
return None
|
||||
return int(s.st_mtime) & 0xffffffff
|
||||
|
||||
def _load_pyc(name, filename, pyc_filename):
|
||||
try:
|
||||
fp = open(pyc_filename, "rb")
|
||||
except IOError:
|
||||
return None
|
||||
if fp.read(4) == imp.get_magic():
|
||||
mtime = struct.unpack('<I', fp.read(4))[0]
|
||||
ptl_mtime = _timestamp(filename)
|
||||
if ptl_mtime is not None and mtime == ptl_mtime:
|
||||
code = marshal.load(fp)
|
||||
return _exec_module_code(code, name, filename)
|
||||
return None
|
||||
|
||||
def _load_ptl(name, filename, file=None):
|
||||
if not file:
|
||||
@classmethod
|
||||
def _path_importer_cache(cls, path):
|
||||
try:
|
||||
file = open(filename, "rb")
|
||||
except IOError:
|
||||
return None
|
||||
path, ext = os.path.splitext(filename)
|
||||
pyc_filename = path + ".pyc"
|
||||
module = _load_pyc(name, filename, pyc_filename)
|
||||
if module is not None:
|
||||
return module
|
||||
try:
|
||||
output = open(pyc_filename, "wb")
|
||||
except IOError:
|
||||
output = None
|
||||
try:
|
||||
code = compile_template(file, filename, output)
|
||||
except:
|
||||
if output:
|
||||
output.close()
|
||||
os.unlink(pyc_filename)
|
||||
raise
|
||||
else:
|
||||
if output:
|
||||
output.close()
|
||||
return _exec_module_code(code, name, filename)
|
||||
finder = cls.path_importer_cache[path]
|
||||
except KeyError:
|
||||
finder = FileFinder(path, (PTLFileLoader, [PTL_EXT]))
|
||||
cls.path_importer_cache[path] = finder
|
||||
return finder
|
||||
|
||||
@classmethod
|
||||
def invalidate_caches(cls):
|
||||
for finder in list(cls.path_importer_cache.values()):
|
||||
if hasattr(finder, 'invalidate_caches'):
|
||||
f = finder.invalidate_caches()
|
||||
|
||||
# Constant used to signal a PTL files
|
||||
PTL_FILE = object()
|
||||
|
||||
class PTLHooks(ihooks.Hooks):
|
||||
|
||||
def get_suffixes(self):
|
||||
# add our suffixes
|
||||
return [(PTL_EXT, 'r', PTL_FILE)] + imp.get_suffixes()
|
||||
|
||||
class PTLLoader(ihooks.ModuleLoader):
|
||||
|
||||
def load_module(self, name, stuff):
|
||||
file, filename, info = stuff
|
||||
(suff, mode, type) = info
|
||||
|
||||
# If it's a PTL file, load it specially.
|
||||
if type is PTL_FILE:
|
||||
return _load_ptl(name, filename, file)
|
||||
|
||||
else:
|
||||
# Otherwise, use the default handler for loading
|
||||
return ihooks.ModuleLoader.load_module(self, name, stuff)
|
||||
|
||||
if sys.hexversion <= 0x20600b0:
|
||||
try:
|
||||
import cimport
|
||||
except ImportError:
|
||||
cimport = None
|
||||
else:
|
||||
# cimport module doesn't handle relative imports
|
||||
cimport = None
|
||||
|
||||
class cModuleImporter(ihooks.ModuleImporter):
|
||||
def __init__(self, loader=None):
|
||||
self.loader = loader or ihooks.ModuleLoader()
|
||||
cimport.set_loader(self.find_import_module)
|
||||
|
||||
def find_import_module(self, fullname, subname, path):
|
||||
stuff = self.loader.find_module(subname, path)
|
||||
if not stuff:
|
||||
return None
|
||||
return self.loader.load_module(fullname, stuff)
|
||||
|
||||
def install(self):
|
||||
self.save_import_module = __builtin__.__import__
|
||||
self.save_reload = __builtin__.reload
|
||||
if not hasattr(__builtin__, 'unload'):
|
||||
__builtin__.unload = None
|
||||
self.save_unload = __builtin__.unload
|
||||
__builtin__.__import__ = cimport.import_module
|
||||
__builtin__.reload = cimport.reload_module
|
||||
__builtin__.unload = self.unload
|
||||
|
||||
_installed = False
|
||||
|
||||
def install():
|
||||
global _installed
|
||||
if not _installed:
|
||||
hooks = PTLHooks()
|
||||
loader = PTLLoader(hooks)
|
||||
if cimport is not None:
|
||||
importer = cModuleImporter(loader)
|
||||
else:
|
||||
importer = ihooks.ModuleImporter(loader)
|
||||
ihooks.install(importer)
|
||||
_installed = True
|
||||
if PTLPathFinder not in sys.meta_path:
|
||||
sys.meta_path.append(PTLPathFinder)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
Loading…
Reference in New Issue