mLinux 4

1 files changed, 1947 insertions, 0 deletions

diff --git a/scripts/lib/bsp/engine.py b/scripts/lib/bsp/engine.py
new file mode 100644
index 0000000..66e2162
--- /dev/null
+++ b/scripts/lib/bsp/engine.py
@@ -0,0 +1,1947 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2012, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# This module implements the templating engine used by 'yocto-bsp' to
+# create BSPs.  The BSP templates are simply the set of files expected
+# to appear in a generated BSP, marked up with a small set of tags
+# used to customize the output.  The engine parses through the
+# templates and generates a Python program containing all the logic
+# and input elements needed to display and retrieve BSP-specific
+# information from the user.  The resulting program uses those results
+# to generate the final BSP files.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] intel.com>
+#
+
+import os
+import sys
+from abc import ABCMeta, abstractmethod
+from tags import *
+import shlex
+import json
+import subprocess
+import shutil
+
+class Line():
+    """
+    Generic (abstract) container representing a line that will appear
+    in the BSP-generating program.
+    """
+    __metaclass__ = ABCMeta
+
+    def __init__(self, line):
+        self.line = line
+        self.generated_line = ""
+        self.prio = sys.maxint
+        self.discard = False
+
+    @abstractmethod
+    def gen(self, context = None):
+        """
+        Generate the final executable line that will appear in the
+        BSP-generation program.
+        """
+        pass
+
+    def escape(self, line):
+        """
+        Escape single and double quotes and backslashes until I find
+        something better (re.escape() escapes way too much).
+        """
+        return line.replace("\\", "\\\\").replace("\"", "\\\"").replace("'", "\\'")
+
+    def parse_error(self, msg, lineno, line):
+         raise SyntaxError("%s: %s" % (msg, line))
+
+
+class NormalLine(Line):
+    """
+    Container for normal (non-tag) lines.
+    """
+    def __init__(self, line):
+        Line.__init__(self, line)
+        self.is_filename = False
+        self.is_dirname = False
+        self.out_filebase = None
+
+    def gen(self, context = None):
+        if self.is_filename:
+            line = "current_file = \"" + os.path.join(self.out_filebase, self.escape(self.line)) + "\"; of = open(current_file, \"w\")"
+        elif self.is_dirname:
+            dirname = os.path.join(self.out_filebase, self.escape(self.line))
+            line = "if not os.path.exists(\"" + dirname + "\"): os.mkdir(\"" + dirname + "\")"
+        else:
+            line = "of.write(\"" + self.escape(self.line) + "\\n\")"
+        return line
+
+
+class CodeLine(Line):
+    """
+    Container for Python code tag lines.
+    """
+    def __init__(self, line):
+        Line.__init__(self, line)
+
+    def gen(self, context = None):
+        return self.line
+
+
+class Assignment:
+    """
+    Representation of everything we know about {{=name }} tags.
+    Instances of these are used by Assignment lines.
+    """
+    def __init__(self, start, end, name):
+        self.start = start
+        self.end = end
+        self.name = name
+
+
+class AssignmentLine(NormalLine):
+    """
+    Container for normal lines containing assignment tags.  Assignment
+    tags must be in ascending order of 'start' value.
+    """
+    def __init__(self, line):
+        NormalLine.__init__(self, line)
+        self.assignments = []
+
+    def add_assignment(self, start, end, name):
+        self.assignments.append(Assignment(start, end, name))
+
+    def gen(self, context = None):
+        line = self.escape(self.line)
+
+        for assignment in self.assignments:
+            replacement = "\" + " + assignment.name + " + \""
+            idx = line.find(ASSIGN_TAG)
+            line = line[:idx] + replacement + line[idx + assignment.end - assignment.start:]
+        if self.is_filename:
+            return "current_file = \"" + os.path.join(self.out_filebase, line) + "\"; of = open(current_file, \"w\")"
+        elif self.is_dirname:
+            dirname = os.path.join(self.out_filebase, line)
+            return "if not os.path.exists(\"" + dirname + "\"): os.mkdir(\"" + dirname + "\")"
+        else:
+            return "of.write(\"" + line + "\\n\")"
+
+
+class InputLine(Line):
+    """
+    Base class for Input lines.
+    """
+    def __init__(self, props, tag, lineno):
+        Line.__init__(self, tag)
+        self.props = props
+        self.lineno = lineno
+
+        try:
+            self.prio = int(props["prio"])
+        except KeyError:
+            self.prio = sys.maxint
+
+    def gen(self, context = None):
+        try:
+            depends_on = self.props["depends-on"]
+            try:
+                depends_on_val = self.props["depends-on-val"]
+            except KeyError:
+                self.parse_error("No 'depends-on-val' for 'depends-on' property",
+                                 self.lineno, self.line)
+        except KeyError:
+            pass
+
+
+class EditBoxInputLine(InputLine):
+    """
+    Base class for 'editbox' Input lines.
+
+    props:
+        name: example - "Load address"
+        msg: example - "Please enter the load address"
+    result:
+        Sets the value of the variable specified by 'name' to
+        whatever the user typed.
+    """
+    def __init__(self, props, tag, lineno):
+        InputLine.__init__(self, props, tag, lineno)
+
+    def gen(self, context = None):
+        InputLine.gen(self, context)
+        name = self.props["name"]
+        if not name:
+            self.parse_error("No input 'name' property found",
+                             self.lineno, self.line)
+        msg = self.props["msg"]
+        if not msg:
+            self.parse_error("No input 'msg' property found",
+                             self.lineno, self.line)
+
+        try:
+            default_choice = self.props["default"]
+        except KeyError:
+            default_choice = ""
+
+        msg += " [default: " + default_choice + "]"
+
+        line = name + " = default(raw_input(\"" + msg + " \"), " + name + ")"
+
+        return line
+
+
+class GitRepoEditBoxInputLine(EditBoxInputLine):
+    """
+    Base class for 'editbox' Input lines for user input of remote git
+    repos.  This class verifies the existence and connectivity of the
+    specified git repo.
+
+    props:
+        name: example - "Load address"
+        msg: example - "Please enter the load address"
+    result:
+        Sets the value of the variable specified by 'name' to
+        whatever the user typed.
+    """
+    def __init__(self, props, tag, lineno):
+        EditBoxInputLine.__init__(self, props, tag, lineno)
+
+    def gen(self, context = None):
+        EditBoxInputLine.gen(self, context)
+        name = self.props["name"]
+        if not name:
+            self.parse_error("No input 'name' property found",
+                             self.lineno, self.line)
+        msg = self.props["msg"]
+        if not msg:
+            self.parse_error("No input 'msg' property found",
+                             self.lineno, self.line)
+
+        try:
+            default_choice = self.props["default"]
+        except KeyError:
+            default_choice = ""
+
+        msg += " [default: " + default_choice + "]"
+
+        line = name + " = get_verified_git_repo(\"" + msg + "\"," + name + ")"
+
+        return line
+
+
+class FileEditBoxInputLine(EditBoxInputLine):
+    """
+    Base class for 'editbox' Input lines for user input of existing
+    files.  This class verifies the existence of the specified file.
+
+    props:
+        name: example - "Load address"
+        msg: example - "Please enter the load address"
+    result:
+        Sets the value of the variable specified by 'name' to
+        whatever the user typed.
+    """
+    def __init__(self, props, tag, lineno):
+        EditBoxInputLine.__init__(self, props, tag, lineno)
+
+    def gen(self, context = None):
+        EditBoxInputLine.gen(self, context)
+        name = self.props["name"]
+        if not name:
+            self.parse_error("No input 'name' property found",
+                             self.lineno, self.line)
+        msg = self.props["msg"]
+        if not msg:
+            self.parse_error("No input 'msg' property found",
+                             self.lineno, self.line)
+
+        try:
+            default_choice = self.props["default"]
+        except KeyError:
+            default_choice = ""
+
+        msg += " [default: " + default_choice + "]"
+
+        line = name + " = get_verified_file(\"" + msg + "\"," + name + ", True)"
+
+        return line
+
+
+class BooleanInputLine(InputLine):
+    """
+    Base class for boolean Input lines.
+    props:
+        name: example - "keyboard"
+        msg:  example - "Got keyboard?"
+    result:
+        Sets the value of the variable specified by 'name' to "yes" or "no"
+        example - keyboard = "yes"
+    """
+    def __init__(self, props, tag, lineno):
+        InputLine.__init__(self, props, tag, lineno)
+
+    def gen(self, context = None):
+        InputLine.gen(self, context)
+        name = self.props["name"]
+        if not name:
+            self.parse_error("No input 'name' property found",
+                             self.lineno, self.line)
+        msg = self.props["msg"]
+        if not msg:
+            self.parse_error("No input 'msg' property found",
+                             self.lineno, self.line)
+
+        try:
+            default_choice = self.props["default"]
+        except KeyError:
+            default_choice = ""
+
+        msg += " [default: " + default_choice + "]"
+
+        line = name + " = boolean(raw_input(\"" + msg + " \"), " + name + ")"
+
+        return line
+
+
+class ListInputLine(InputLine):
+    """
+    Base class for List-based Input lines. e.g. Choicelist, Checklist.
+    """
+    __metaclass__ = ABCMeta
+
+    def __init__(self, props, tag, lineno):
+        InputLine.__init__(self, props, tag, lineno)
+        self.choices = []
+
+    def gen_choicepair_list(self):
+        """Generate a list of 2-item val:desc lists from self.choices."""
+        if not self.choices:
+            return None
+
+        choicepair_list = list()
+
+        for choice in self.choices:
+            choicepair = []
+            choicepair.append(choice.val)
+            choicepair.append(choice.desc)
+            choicepair_list.append(choicepair)
+
+        return choicepair_list
+
+    def gen_degenerate_choicepair_list(self, choices):
+        """Generate a list of 2-item val:desc with val=desc from passed-in choices."""
+        choicepair_list = list()
+
+        for choice in choices:
+            choicepair = []
+            choicepair.append(choice)
+            choicepair.append(choice)
+            choicepair_list.append(choicepair)
+
+        return choicepair_list
+
+    def exec_listgen_fn(self, context = None):
+        """
+        Execute the list-generating function contained as a string in
+        the "gen" property.
+        """
+        retval = None
+        try:
+            fname = self.props["gen"]
+            modsplit = fname.split('.')
+            mod_fn = modsplit.pop()
+            mod = '.'.join(modsplit)
+
+            __import__(mod)
+            # python 2.7 has a better way to do this using importlib.import_module
+            m = sys.modules[mod]
+
+            fn = getattr(m, mod_fn)
+            if not fn:
+                self.parse_error("couldn't load function specified for 'gen' property ",
+                                 self.lineno, self.line)
+            retval = fn(context)
+            if not retval:
+                self.parse_error("function specified for 'gen' property returned nothing ",
+                                 self.lineno, self.line)
+        except KeyError:
+            pass
+
+        return retval
+
+    def gen_choices_str(self, choicepairs):
+        """
+        Generate a numbered list of choices from a list of choicepairs
+        for display to the user.
+        """
+        choices_str = ""
+
+        for i, choicepair in enumerate(choicepairs):
+            choices_str += "\t" + str(i + 1) + ") " + choicepair[1] + "\n"
+
+        return choices_str
+
+    def gen_choices_val_str(self, choicepairs):
+        """
+        Generate an array of choice values corresponding to the
+        numbered list generated by gen_choices_str().
+        """
+        choices_val_list = "["
+
+        for i, choicepair in enumerate(choicepairs):
+            choices_val_list += "\"" + choicepair[0] + "\","
+        choices_val_list += "]"
+
+        return choices_val_list
+
+    def gen_choices_val_list(self, choicepairs):
+        """
+        Generate an array of choice values corresponding to the
+        numbered list generated by gen_choices_str().
+        """
+        choices_val_list = []
+
+        for i, choicepair in enumerate(choicepairs):
+            choices_val_list.append(choicepair[0])
+
+        return choices_val_list
+
+    def gen_choices_list(self, context = None, checklist = False):
+        """
+        Generate an array of choice values corresponding to the
+        numbered list generated by gen_choices_str().
+        """
+        choices = self.exec_listgen_fn(context)
+        if choices:
+            if len(choices) == 0:
+                self.parse_error("No entries available for input list",
+                                 self.lineno, self.line)
+            choicepairs = self.gen_degenerate_choicepair_list(choices)
+        else:
+            if len(self.choices) == 0:
+                self.parse_error("No entries available for input list",
+                                 self.lineno, self.line)
+            choicepairs = self.gen_choicepair_list()
+
+        return choicepairs
+
+    def gen_choices(self, context = None, checklist = False):
+        """
+        Generate an array of choice values corresponding to the
+        numbered list generated by gen_choices_str(), display it to
+        the user, and process the result.
+        """
+        msg = self.props["msg"]
+        if not msg:
+            self.parse_error("No input 'msg' property found",
+                             self.lineno, self.line)
+
+        try:
+            default_choice = self.props["default"]
+        except KeyError:
+            default_choice = ""
+
+        msg += " [default: " + default_choice + "]"
+
+        choicepairs = self.gen_choices_list(context, checklist)
+
+        choices_str = self.gen_choices_str(choicepairs)
+        choices_val_list = self.gen_choices_val_list(choicepairs)
+        if checklist:
+            choiceval = default(find_choicevals(raw_input(msg + "\n" + choices_str), choices_val_list), default_choice)
+        else:
+            choiceval = default(find_choiceval(raw_input(msg + "\n" + choices_str), choices_val_list), default_choice)
+
+        return choiceval
+
+
+def find_choiceval(choice_str, choice_list):
+    """
+    Take number as string and return val string from choice_list,
+    empty string if oob.  choice_list is a simple python list.
+    """
+    choice_val = ""
+
+    try:
+        choice_idx = int(choice_str)
+        if choice_idx <= len(choice_list):
+            choice_idx -= 1
+            choice_val = choice_list[choice_idx]
+    except ValueError:
+        pass
+
+    return choice_val
+
+
+def find_choicevals(choice_str, choice_list):
+    """
+    Take numbers as space-separated string and return vals list from
+    choice_list, empty list if oob.  choice_list is a simple python
+    list.
+    """
+    choice_vals = []
+
+    choices = choice_str.split()
+    for choice in choices:
+        choice_vals.append(find_choiceval(choice, choice_list))
+
+    return choice_vals
+
+
+def default(input_str, name):
+    """
+    Return default if no input_str, otherwise stripped input_str.
+    """
+    if not input_str:
+        return name
+
+    return input_str.strip()
+
+
+def verify_git_repo(giturl):
+    """
+    Verify that the giturl passed in can be connected to.  This can be
+    used as a check for the existence of the given repo and/or basic
+    git remote connectivity.
+
+    Returns True if the connection was successful, fals otherwise
+    """
+    if not giturl:
+        return False
+
+    gitcmd = "git ls-remote %s > /dev/null 2>&1" % (giturl)
+    rc = subprocess.call(gitcmd, shell=True)
+    if rc == 0:
+        return True
+
+    return False
+
+
+def get_verified_git_repo(input_str, name):
+    """
+    Return git repo if verified, otherwise loop forever asking user
+    for filename.
+    """
+    msg = input_str.strip() + " "
+
+    giturl = default(raw_input(msg), name)
+
+    while True:
+        if verify_git_repo(giturl):
+            return giturl
+        giturl = default(raw_input(msg), name)
+
+
+def get_verified_file(input_str, name, filename_can_be_null):
+    """
+    Return filename if the file exists, otherwise loop forever asking
+    user for filename.
+    """
+    msg = input_str.strip() + " "
+
+    filename = default(raw_input(msg), name)
+
+    while True:
+        if not filename and filename_can_be_null:
+            return filename
+        if os.path.isfile(filename):
+            return filename
+        filename = default(raw_input(msg), name)
+
+
+def replace_file(replace_this, with_this):
+    """
+    Replace the given file with the contents of filename, retaining
+    the original filename.
+    """
+    try:
+        replace_this.close()
+        shutil.copy(with_this, replace_this.name)
+    except IOError:
+        pass
+
+
+def boolean(input_str, name):
+    """
+    Return lowercase version of first char in string, or value in name.
+    """
+    if not input_str:
+        return name
+
+    str = input_str.lower().strip()
+    if str and str[0] == "y" or str[0] == "n":
+        return str[0]
+    else:
+        return name
+
+
+def strip_base(input_str):
+    """
+    strip '/base' off the end of input_str, so we can use 'base' in
+    the branch names we present to the user.
+    """
+    if input_str and input_str.endswith("/base"):
+        return input_str[:-len("/base")]
+    return input_str.strip()
+
+
+deferred_choices = {}
+
+def gen_choices_defer(input_line, context, checklist = False):
+    """
+    Save the context hashed the name of the input item, which will be
+    passed to the gen function later.
+    """
+    name = input_line.props["name"]
+
+    try:
+        nameappend = input_line.props["nameappend"]
+    except KeyError:
+        nameappend = ""
+
+    try:
+        branches_base = input_line.props["branches_base"]
+    except KeyError:
+        branches_base = ""
+
+    filename = input_line.props["filename"]
+
+    closetag_start = filename.find(CLOSE_TAG)
+
+    if closetag_start != -1:
+        filename = filename[closetag_start + len(CLOSE_TAG):]
+
+    filename = filename.strip()
+    filename = os.path.splitext(filename)[0]
+
+    captured_context = capture_context(context)
+    context["filename"] = filename
+    captured_context["filename"] = filename
+    context["nameappend"] = nameappend
+    captured_context["nameappend"] = nameappend
+    context["branches_base"] = branches_base
+    captured_context["branches_base"] = branches_base
+
+    deferred_choice = (input_line, captured_context, checklist)
+    key = name + "_" + filename + "_" + nameappend
+    deferred_choices[key] = deferred_choice
+
+
+def invoke_deferred_choices(name):
+    """
+    Invoke the choice generation function using the context hashed by
+    'name'.
+    """
+    deferred_choice = deferred_choices[name]
+    input_line = deferred_choice[0]
+    context = deferred_choice[1]
+    checklist = deferred_choice[2]
+
+    context["name"] = name
+
+    choices = input_line.gen_choices(context, checklist)
+
+    return choices
+
+
+class ChoicelistInputLine(ListInputLine):
+    """
+    Base class for choicelist Input lines.
+    props:
+        name: example - "xserver_choice"
+        msg:  example - "Please select an xserver for this machine"
+    result:
+        Sets the value of the variable specified by 'name' to whichever Choice was chosen
+        example - xserver_choice = "xserver_vesa"
+    """
+    def __init__(self, props, tag, lineno):
+        ListInputLine.__init__(self, props, tag, lineno)
+
+    def gen(self, context = None):
+        InputLine.gen(self, context)
+
+        gen_choices_defer(self, context)
+        name = self.props["name"]
+        nameappend = context["nameappend"]
+        filename = context["filename"]
+
+        try:
+            default_choice = self.props["default"]
+        except KeyError:
+            default_choice = ""
+
+        line = name + " = default(invoke_deferred_choices(\"" + name + "_" + filename + "_" + nameappend + "\"), \"" + default_choice + "\")"
+
+        return line
+
+
+class ListValInputLine(InputLine):
+    """
+    Abstract base class for choice and checkbox Input lines.
+    """
+    def __init__(self, props, tag, lineno):
+        InputLine.__init__(self, props, tag, lineno)
+
+        try:
+            self.val = self.props["val"]
+        except KeyError:
+            self.parse_error("No input 'val' property found", self.lineno, self.line)
+
+        try:
+            self.desc = self.props["msg"]
+        except KeyError:
+            self.parse_error("No input 'msg' property found", self.lineno, self.line)
+
+
+class ChoiceInputLine(ListValInputLine):
+    """
+    Base class for choicelist item Input lines.
+    """
+    def __init__(self, props, tag, lineno):
+        ListValInputLine.__init__(self, props, tag, lineno)
+
+    def gen(self, context = None):
+        return None
+
+
+class ChecklistInputLine(ListInputLine):
+    """
+    Base class for checklist Input lines.
+    """
+    def __init__(self, props, tag, lineno):
+        ListInputLine.__init__(self, props, tag, lineno)
+
+    def gen(self, context = None):
+        InputLine.gen(self, context)
+
+        gen_choices_defer(self, context, True)
+        name = self.props["name"]
+        nameappend = context["nameappend"]
+        filename = context["filename"]
+
+        try:
+            default_choice = self.props["default"]
+        except KeyError:
+            default_choice = ""
+
+        line = name + " = default(invoke_deferred_choices(\"" + name + "_" + filename + "_" + nameappend + "\"), \"" + default_choice + "\")"
+
+        return line
+
+
+class CheckInputLine(ListValInputLine):
+    """
+    Base class for checklist item Input lines.
+    """
+    def __init__(self, props, tag, lineno):
+        ListValInputLine.__init__(self, props, tag, lineno)
+
+    def gen(self, context = None):
+        return None
+
+
+dirname_substitutions = {}
+
+class SubstrateBase(object):
+    """
+    Base class for both expanded and unexpanded file and dir container
+    objects.
+    """
+    def __init__(self, filename, filebase, out_filebase):
+        self.filename = filename
+        self.filebase = filebase
+        self.translated_filename = filename
+        self.out_filebase = out_filebase
+        self.raw_lines = []
+        self.expanded_lines = []
+        self.prev_choicelist = None
+
+    def parse_error(self, msg, lineno, line):
+         raise SyntaxError("%s: [%s: %d]: %s" % (msg, self.filename, lineno, line))
+
+    def expand_input_tag(self, tag, lineno):
+        """
+        Input tags consist of the word 'input' at the beginning,
+        followed by name:value property pairs which are converted into
+        a dictionary.
+        """
+        propstr = tag[len(INPUT_TAG):]
+
+        props = dict(prop.split(":", 1) for prop in shlex.split(propstr))
+        props["filename"] = self.filename
+
+        input_type = props[INPUT_TYPE_PROPERTY]
+        if not props[INPUT_TYPE_PROPERTY]:
+            self.parse_error("No input 'type' property found", lineno, tag)
+
+        if input_type == "boolean":
+            return BooleanInputLine(props, tag, lineno)
+        if input_type == "edit":
+            return EditBoxInputLine(props, tag, lineno)
+        if input_type == "edit-git-repo":
+            return GitRepoEditBoxInputLine(props, tag, lineno)
+        if input_type == "edit-file":
+            return FileEditBoxInputLine(props, tag, lineno)
+        elif input_type == "choicelist":
+            self.prev_choicelist = ChoicelistInputLine(props, tag, lineno)
+            return self.prev_choicelist
+        elif input_type == "choice":
+            if not self.prev_choicelist:
+                self.parse_error("Found 'choice' input tag but no previous choicelist",
+                                 lineno, tag)
+            choice = ChoiceInputLine(props, tag, lineno)
+            self.prev_choicelist.choices.append(choice)
+            return choice
+        elif input_type == "checklist":
+            return ChecklistInputLine(props, tag, lineno)
+        elif input_type == "check":
+            return CheckInputLine(props, tag, lineno)
+
+    def expand_assignment_tag(self, start, line, lineno):
+        """
+        Expand all tags in a line.
+        """
+        expanded_line = AssignmentLine(line.rstrip())
+
+        while start != -1:
+            end = line.find(CLOSE_TAG, start)
+            if end == -1:
+                self.parse_error("No close tag found for assignment tag", lineno, line)
+            else:
+                name = line[start + len(ASSIGN_TAG):end].strip()
+                expanded_line.add_assignment(start, end + len(CLOSE_TAG), name)
+                start = line.find(ASSIGN_TAG, end)
+
+        return expanded_line
+
+    def expand_tag(self, line, lineno):
+        """
+        Returns a processed tag line, or None if there was no tag
+
+        The rules for tags are very simple:
+            - No nested tags
+            - Tags start with {{ and end with }}
+            - An assign tag, {{=, can appear anywhere and will
+              be replaced with what the assignment evaluates to
+            - Any other tag occupies the whole line it is on
+                - if there's anything else on the tag line, it's an error
+                - if it starts with 'input', it's an input tag and
+                  will only be used for prompting and setting variables
+                - anything else is straight Python
+                - tags are in effect only until the next blank line or tag or 'pass' tag
+                - we don't have indentation in tags, but we need some way to end a block
+                  forcefully without blank lines or other tags - that's the 'pass' tag
+                - todo: implement pass tag
+                - directories and filenames can have tags as well, but only assignment
+                  and 'if' code lines
+                - directories and filenames are the only case where normal tags can
+                  coexist with normal text on the same 'line'
+        """
+        start = line.find(ASSIGN_TAG)
+        if start != -1:
+            return self.expand_assignment_tag(start, line, lineno)
+
+        start = line.find(OPEN_TAG)
+        if start == -1:
+            return None
+
+        end = line.find(CLOSE_TAG, 0)
+        if end == -1:
+             self.parse_error("No close tag found for open tag", lineno, line)
+
+        tag = line[start + len(OPEN_TAG):end].strip()
+
+        if not tag.lstrip().startswith(INPUT_TAG):
+            return CodeLine(tag)
+
+        return self.expand_input_tag(tag, lineno)
+
+    def append_translated_filename(self, filename):
+        """
+        Simply append filename to translated_filename
+        """
+        self.translated_filename = os.path.join(self.translated_filename, filename)
+
+    def get_substituted_file_or_dir_name(self, first_line, tag):
+        """
+        If file or dir names contain name substitutions, return the name
+        to substitute.  Note that this is just the file or dirname and
+        doesn't include the path.
+        """
+        filename = first_line.find(tag)
+        if filename != -1:
+            filename += len(tag)
+        substituted_filename = first_line[filename:].strip()
+        this = substituted_filename.find(" this")
+        if this != -1:
+            head, tail = os.path.split(self.filename)
+            substituted_filename = substituted_filename[:this + 1] + tail
+            if tag == DIRNAME_TAG: # get rid of .noinstall in dirname
+                substituted_filename = substituted_filename.split('.')[0]
+
+        return substituted_filename
+
+    def get_substituted_filename(self, first_line):
+        """
+        If a filename contains a name substitution, return the name to
+        substitute.  Note that this is just the filename and doesn't
+        include the path.
+        """
+        return self.get_substituted_file_or_dir_name(first_line, FILENAME_TAG)
+
+    def get_substituted_dirname(self, first_line):
+        """
+        If a dirname contains a name substitution, return the name to
+        substitute.  Note that this is just the dirname and doesn't
+        include the path.
+        """
+        return self.get_substituted_file_or_dir_name(first_line, DIRNAME_TAG)
+
+    def substitute_filename(self, first_line):
+        """
+        Find the filename in first_line and append it to translated_filename.
+        """
+        substituted_filename = self.get_substituted_filename(first_line)
+        self.append_translated_filename(substituted_filename);
+
+    def substitute_dirname(self, first_line):
+        """
+        Find the dirname in first_line and append it to translated_filename.
+        """
+        substituted_dirname = self.get_substituted_dirname(first_line)
+        self.append_translated_filename(substituted_dirname);
+
+    def is_filename_substitution(self, line):
+        """
+        Do we have a filename subustition?
+        """
+        if line.find(FILENAME_TAG) != -1:
+            return True
+        return False
+
+    def is_dirname_substitution(self, line):
+        """
+        Do we have a dirname subustition?
+        """
+        if line.find(DIRNAME_TAG) != -1:
+            return True
+        return False
+
+    def translate_dirname(self, first_line):
+        """
+        Just save the first_line mapped by filename.  The later pass
+        through the directories will look for a dirname.noinstall
+        match and grab the substitution line.
+        """
+        dirname_substitutions[self.filename] = first_line
+
+    def translate_dirnames_in_path(self, path):
+        """
+        Translate dirnames below this file or dir, not including tail.
+        dirname_substititions is keyed on actual untranslated filenames.
+        translated_path contains the subsititutions for each element.
+        """
+        remainder = path[len(self.filebase)+1:]
+        translated_path = untranslated_path = self.filebase
+
+        untranslated_dirs = remainder.split(os.sep)
+
+        for dir in untranslated_dirs:
+            key = os.path.join(untranslated_path, dir + '.noinstall')
+            try:
+                first_line = dirname_substitutions[key]
+            except KeyError:
+                translated_path = os.path.join(translated_path, dir)
+                untranslated_path = os.path.join(untranslated_path, dir)
+                continue
+            substituted_dir = self.get_substituted_dirname(first_line)
+            translated_path = os.path.join(translated_path, substituted_dir)
+            untranslated_path = os.path.join(untranslated_path, dir)
+
+        return translated_path
+
+    def translate_file_or_dir_name(self):
+        """
+        Originally we were allowed to use open/close/assign tags and python
+        code in the filename, which fit in nicely with the way we
+        processed the templates and generated code.  Now that we can't
+        do that, we make those tags proper file contents and have this
+        pass substitute the nice but non-functional names with those
+        'strange' ones, and then proceed as usual.
+
+        So, if files or matching dir<.noinstall> files contain
+        filename substitutions, this function translates them into the
+        corresponding 'strange' names, which future passes will expand
+        as they always have.  The resulting pathname is kept in the
+        file or directory's translated_filename.  Another way to think
+        about it is that self.filename is the input filename, and
+        translated_filename is the output filename before expansion.
+        """
+        # remove leaf file or dirname
+        head, tail = os.path.split(self.filename)
+        translated_path = self.translate_dirnames_in_path(head)
+        self.translated_filename = translated_path
+
+        # This is a dirname - does it have a matching .noinstall with
+        # a substitution?  If so, apply the dirname subsititution.
+        if not os.path.isfile(self.filename):
+            key = self.filename + ".noinstall"
+            try:
+                first_line = dirname_substitutions[key]
+            except KeyError:
+                self.append_translated_filename(tail)
+                return
+            self.substitute_dirname(first_line)
+            return
+
+        f = open(self.filename)
+        first_line = f.readline()
+        f.close()
+
+        # This is a normal filename not needing translation, just use
+        # it as-is.
+        if not first_line or not first_line.startswith("#"):
+            self.append_translated_filename(tail)
+            return
+
+        # If we have a filename substitution (first line in the file
+        # is a FILENAME_TAG line) do the substitution now.  If we have
+        # a dirname substitution (DIRNAME_TAG in dirname.noinstall
+        # meta-file), hash it so we can apply it when we see the
+        # matching dirname later.  Otherwise we have a regular
+        # filename, just use it as-is.
+        if self.is_filename_substitution(first_line):
+            self.substitute_filename(first_line)
+        elif self.is_dirname_substitution(first_line):
+            self.translate_dirname(first_line)
+        el