diff --git a/quixote/ptl/ptl_compile.py b/quixote/ptl/ptl_compile.py index 31aeadf..533d572 100644 --- a/quixote/ptl/ptl_compile.py +++ b/quixote/ptl/ptl_compile.py @@ -5,37 +5,27 @@ First template function names are mangled, noting the template type. Next, the file is parsed into a parse tree. This tree is converted into a modified AST. It is during this state that the semantics are modified by adding extra nodes to the tree. Finally bytecode is generated using -the compiler package. +the compiler. """ +import ast import sys import os import stat -import symbol -import token import re import imp -import stat import marshal import struct -assert sys.hexversion >= 0x20300b1, 'PTL requires Python 2.3 or newer' - -from compiler import pycodegen, transformer -from compiler import ast -from compiler.consts import OP_ASSIGN -from compiler import misc, syntax - HTML_TEMPLATE_PREFIX = "_q_html_template_" PLAIN_TEMPLATE_PREFIX = "_q_plain_template_" -class TemplateTransformer(transformer.Transformer): - +class TemplateTransformer(ast.NodeTransformer): def __init__(self, *args, **kwargs): - transformer.Transformer.__init__(self, *args, **kwargs) + ast.NodeTransformer.__init__(self, *args, **kwargs) # __template_type is a stack whose values are # "html", "plain", or None - self.__template_type = [] + self.__template_type = [None] def _get_template_type(self): """Return the type of the function being compiled ( @@ -46,129 +36,93 @@ class TemplateTransformer(transformer.Transformer): else: return None - def file_input(self, nodelist): - doc = None # self.get_docstring(nodelist, symbol.file_input) - if sys.hexversion >= 0x02050000: - html_imp = ast.From( - 'quixote.html', - [('TemplateIO', '_q_TemplateIO'), ('htmltext', '_q_htmltext')], - 0) - vars_imp = ast.From("__builtin__", [("vars", "_q_vars")], 0) - else: - html_imp = ast.From( - 'quixote.html', - [('TemplateIO', '_q_TemplateIO'), ('htmltext', '_q_htmltext')]) - vars_imp = ast.From("__builtin__", [("vars", "_q_vars")]) - - ptl_imports = [ vars_imp, html_imp ] - stmts = [] - for node in nodelist: - if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: - self.com_append_stmt(stmts, node) + def visit_Module(self, node): + html_imp = ast.ImportFrom(module='quixote.html', + names=[ast.alias(name='TemplateIO', + asname='_q_TemplateIO'), + ast.alias(name='htmltext', + asname='_q_htmltext')], + level=0) + ast.fix_missing_locations(html_imp) + vars_imp = ast.ImportFrom(module='builtins', + names=[ast.alias(name='vars', + asname='_q_vars')], level=0) + ast.fix_missing_locations(vars_imp) + ptl_imports = [vars_imp, html_imp] # count __future__ statements i = 0 - for stmt in stmts: - if isinstance(stmt, ast.From) and stmt.modname == '__future__': + for stmt in node.body: + if isinstance(stmt, ast.ImportFrom) and stmt.module == '__future__': i += 1 else: break - stmts[i:i] = ptl_imports - return ast.Module(doc, ast.Stmt(stmts)) + node.body[i:i] = ptl_imports + return self.generic_visit(node) - def funcdef(self, nodelist): - if len(nodelist) == 6: - assert nodelist[0][0] == symbol.decorators - decorators = self.decorators(nodelist[0][1:]) - else: - assert len(nodelist) == 5 - decorators = None - - lineno = nodelist[-4][2] - name = nodelist[-4][1] - args = nodelist[-3][2] - - if not re.match('_q_(html|plain)_(dollar_)?template_', name): - # just a normal function, let base class handle it + def visit_FunctionDef(self, node): + name = node.name + if not re.match('_q_(html|plain)_template_', name): + # just a normal function self.__template_type.append(None) - n = transformer.Transformer.funcdef(self, nodelist) + node = self.generic_visit(node) else: if name.startswith(PLAIN_TEMPLATE_PREFIX): - name = name[len(PLAIN_TEMPLATE_PREFIX):] + node.name = name[len(PLAIN_TEMPLATE_PREFIX):] template_type = "plain" elif name.startswith(HTML_TEMPLATE_PREFIX): - name = name[len(HTML_TEMPLATE_PREFIX):] + node.name = name[len(HTML_TEMPLATE_PREFIX):] template_type = "html" else: - raise RuntimeError, 'unknown prefix on %s' % name + raise RuntimeError('unknown prefix on %s' % name) self.__template_type.append(template_type) - - if args[0] == symbol.varargslist: - names, defaults, flags = self.com_arglist(args[1:]) - else: - names = defaults = () - flags = 0 - doc = None # self.get_docstring(nodelist[-1]) - - # code for function - code = self.com_node(nodelist[-1]) + node = self.generic_visit(node) # _q_output = _q_TemplateIO() - klass = ast.Name('_q_TemplateIO') - args = [ast.Const(template_type == "html")] - instance = ast.CallFunc(klass, args) - assign_name = ast.AssName('_q_output', OP_ASSIGN) - assign = ast.Assign([assign_name], instance) + klass = ast.Name(id='_q_TemplateIO', ctx=ast.Load()) + arg = ast.NameConstant(template_type == 'html') + instance = ast.Call(func=klass, args=[arg], keywords=[], + starargs=None, kwargs=None) + assign_name = ast.Name(id='_q_output', ctx=ast.Store()) + assign = ast.Assign(targets=[assign_name], value=instance) + ast.copy_location(assign, node) + ast.fix_missing_locations(assign) + node.body.insert(0, assign) # return _q_output.getvalue() - func = ast.Getattr(ast.Name('_q_output'), "getvalue") - ret = ast.Return(ast.CallFunc(func, [])) - - # wrap original function code - code = ast.Stmt([assign, code, ret]) - code.lineno = lineno - - if sys.hexversion >= 0x20400a2: - n = ast.Function(decorators, name, names, defaults, flags, doc, - code) - else: - n = ast.Function(name, names, defaults, flags, doc, code) - n.lineno = lineno + n = ast.Name(id='_q_output', ctx=ast.Load()) + n = ast.Attribute(value=n, attr='getvalue', ctx=ast.Load()) + n = ast.Call(func=n, args=[], keywords=[], starargs=None, + kwargs=None) + ret = ast.Return(value=n) + ast.copy_location(ret, node.body[-1]) + ast.fix_missing_locations(ret) + node.body.append(ret) self.__template_type.pop() - return n + return node - def expr_stmt(self, nodelist): - if self._get_template_type() is None: - return transformer.Transformer.expr_stmt(self, nodelist) - - # Instead of discarding objects on the stack, call - # "_q_output += obj". - exprNode = self.com_node(nodelist[-1]) - if len(nodelist) == 1: - lval = ast.Name('_q_output') - n = ast.AugAssign(lval, '+=', exprNode) - if hasattr(exprNode, 'lineno'): - n.lineno = exprNode.lineno - elif nodelist[1][0] == token.EQUAL: - nodes = [ ] - for i in range(0, len(nodelist) - 2, 2): - nodes.append(self.com_assign(nodelist[i], OP_ASSIGN)) - n = ast.Assign(nodes, exprNode) - n.lineno = nodelist[1][2] + def visit_Expr(self, node): + if self._get_template_type() is not None: + node = self.generic_visit(node) + # Instead of discarding objects on the stack, call + # "_q_output += obj". + lval = ast.Name(id='_q_output', ctx=ast.Store()) + ast.copy_location(lval, node) + aug = ast.AugAssign(target=lval, op=ast.Add(), value=node.value) + return ast.copy_location(aug, node) else: - lval = self.com_augassign(nodelist[0]) - op = self.com_augassign_op(nodelist[1]) - n = ast.AugAssign(lval, op[1], exprNode) - n.lineno = op[2] - return n + return node - def atom_string(self, nodelist): - const_node = transformer.Transformer.atom_string(self, nodelist) + def visit_Str(self, node): if "html" == self._get_template_type(): - return ast.CallFunc(ast.Name('_q_htmltext'), [const_node]) + n = ast.Name(id='_q_htmltext', ctx=ast.Load()) + ast.copy_location(n, node) + n = ast.Call(func=n, args=[node], keywords=[], starargs=None, + kwargs=None) + return ast.copy_location(n, node) else: - return const_node + return node _template_re = re.compile( r"^(?P[ \t]*) def (?:[ \t]+)" @@ -200,31 +154,26 @@ def translate_tokens(buf): def parse(buf, filename=''): buf = translate_tokens(buf) try: - return TemplateTransformer().parsesuite(buf) - except SyntaxError, e: + node = ast.parse(buf, filename) + except SyntaxError as e: # set the filename attribute raise SyntaxError(str(e), (filename, e.lineno, e.offset, e.text)) + t = TemplateTransformer() + return t.visit(node) PTL_EXT = ".ptl" -class Template(pycodegen.Module): - - def _get_tree(self): - tree = parse(self.source, self.filename) - misc.set_filename(self.filename, tree) - syntax.check(tree) - return tree - - def dump(self, fp): - mtime = os.stat(self.filename)[stat.ST_MTIME] - fp.write('\0\0\0\0') - fp.write(struct.pack(' code @@ -233,12 +182,11 @@ def compile_template(input, filename, output=None): If output is not None then the code is written to output. The code object is returned. """ - buf = input.read() - template = Template(buf, filename) - template.compile() + node = parse(input.read(), filename) + code = _compile(node, filename, 'exec') if output is not None: - template.dump(output) - return template.code + dump(code, filename, output) + return code def compile(inputname, outputname): """(inputname, outputname) @@ -266,7 +214,7 @@ def compile_file(filename, force=0, verbose=0): if (ctime > ftime) and not force: return if verbose: - print 'Compiling', filename, '...' + print('Compiling', filename, '...') ok = compile(filename, cfile) def compile_dir(dir, maxlevels=10, force=0): @@ -279,11 +227,11 @@ def compile_dir(dir, maxlevels=10, force=0): maxlevels: maximum recursion level (default 10) force: if true, force compilation, even if timestamps are up-to-date """ - print 'Listing', dir, '...' + print('Listing', dir, '...') try: names = os.listdir(dir) except os.error: - print "Can't list", dir + print("Can't list", dir) names = [] names.sort() success = 1 @@ -299,8 +247,8 @@ def compile_dir(dir, maxlevels=10, force=0): if type(sys.exc_type) == type(''): exc_type_name = sys.exc_type else: exc_type_name = sys.exc_type.__name__ - print 'Sorry:', exc_type_name + ':', - print sys.exc_value + print('Sorry:', exc_type_name + ':',) + print(sys.exc_value) success = 0 else: if ok == 0: @@ -327,7 +275,7 @@ def compile_package(path, force=0, verbose=0): def main(): args = sys.argv[1:] if not args: - print "no files to compile" + print("no files to compile") else: for filename in args: path, ext = os.path.splitext(filename) diff --git a/quixote/ptl/ptl_import.py b/quixote/ptl/ptl_import.py index 8582ffd..90d1947 100644 --- a/quixote/ptl/ptl_import.py +++ b/quixote/ptl/ptl_import.py @@ -1,156 +1,44 @@ """Import hooks; when installed, these hooks allow importing .ptl files as if they were Python modules. - -Note: there's some unpleasant incompatibility between ZODB's import -trickery and the import hooks here. Bottom line: if you're using ZODB, -import it *before* installing the PTL import hooks. """ import sys -import os.path -import imp, ihooks, new -import struct -import marshal -import __builtin__ +from importlib.machinery import FileFinder, PathFinder, SourceFileLoader -# Check for a deficient ihooks module. Python 2.6 was released without -# ihooks.py being updated to support relative imports. Any library that uses -# relative imports will cause the import hook to fail. Use our local copy of -# ihooks module which does have support for relative imports. -if sys.hexversion >= 0x20600b0: - _m = ihooks.ModuleImporter.import_module - if _m.im_func.func_code.co_argcount == 5: - import ihooks_local as ihooks +from quixote.ptl.ptl_compile import parse, PTL_EXT -from quixote.ptl.ptl_compile import compile_template, PTL_EXT +class PTLFileLoader(SourceFileLoader): + @staticmethod + def source_to_code(data, path=''): + if isinstance(data, bytes): + # FIXME: we should check the encoding of the source file + data = data.decode('utf-8') + node = parse(data, path) + return compile(node, path, 'exec') -assert sys.hexversion >= 0x20000b1, "need Python 2.0b1 or later" -def _exec_module_code(code, name, filename): - if name in sys.modules: - mod = sys.modules[name] # necessary for reload() - else: - mod = new.module(name) - sys.modules[name] = mod - mod.__name__ = name - mod.__file__ = filename - exec code in mod.__dict__ - return mod +class PTLPathFinder(PathFinder): + path_importer_cache = {} -def _timestamp(filename): - try: - s = os.stat(filename) - except OSError: - return None - return int(s.st_mtime) & 0xffffffff - -def _load_pyc(name, filename, pyc_filename): - try: - fp = open(pyc_filename, "rb") - except IOError: - return None - if fp.read(4) == imp.get_magic(): - mtime = struct.unpack('