diff options
| author | Chris Larson <chris_larson@mentor.com> | 2010-08-02 13:42:23 -0700 |
|---|---|---|
| committer | Richard Purdie <rpurdie@linux.intel.com> | 2010-08-03 14:06:07 +0100 |
| commit | 191a2883492841f30bbc21ab7bf4e4a0810d0760 (patch) | |
| tree | f91e66f57380a47ea31e95bac28a93dd1e9e6370 /bitbake/lib/pysh | |
| parent | e6b6767369f6d0caa7a9efe32ccd4ed514bb3148 (diff) | |
| download | openembedded-core-191a2883492841f30bbc21ab7bf4e4a0810d0760.tar.gz openembedded-core-191a2883492841f30bbc21ab7bf4e4a0810d0760.tar.bz2 openembedded-core-191a2883492841f30bbc21ab7bf4e4a0810d0760.zip | |
Add pysh, ply, and codegen to lib/ to prepare for future work
(Bitbake rev: d0a6e9c5c1887a885e0e73eba264ca66801f5ed0)
Signed-off-by: Chris Larson <chris_larson@mentor.com>
Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
Diffstat (limited to 'bitbake/lib/pysh')
| -rw-r--r-- | bitbake/lib/pysh/__init__.py | 0 | ||||
| -rw-r--r-- | bitbake/lib/pysh/builtin.py | 710 | ||||
| -rw-r--r-- | bitbake/lib/pysh/interp.py | 1367 | ||||
| -rw-r--r-- | bitbake/lib/pysh/lsprof.py | 116 | ||||
| -rw-r--r-- | bitbake/lib/pysh/pysh.py | 167 | ||||
| -rw-r--r-- | bitbake/lib/pysh/pyshlex.py | 888 | ||||
| -rw-r--r-- | bitbake/lib/pysh/pyshyacc.py | 772 | ||||
| -rw-r--r-- | bitbake/lib/pysh/sherrors.py | 41 | ||||
| -rw-r--r-- | bitbake/lib/pysh/subprocess_fix.py | 77 |
9 files changed, 4138 insertions, 0 deletions
diff --git a/bitbake/lib/pysh/__init__.py b/bitbake/lib/pysh/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/bitbake/lib/pysh/__init__.py diff --git a/bitbake/lib/pysh/builtin.py b/bitbake/lib/pysh/builtin.py new file mode 100644 index 0000000000..25ad22eb74 --- /dev/null +++ b/bitbake/lib/pysh/builtin.py @@ -0,0 +1,710 @@ +# builtin.py - builtins and utilities definitions for pysh. +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +"""Builtin and internal utilities implementations. + +- Beware not to use python interpreter environment as if it were the shell +environment. For instance, commands working directory must be explicitely handled +through env['PWD'] instead of relying on python working directory. +""" +import errno +import optparse +import os +import re +import subprocess +import sys +import time + +def has_subprocess_bug(): + return getattr(subprocess, 'list2cmdline') and \ + ( subprocess.list2cmdline(['']) == '' or \ + subprocess.list2cmdline(['foo|bar']) == 'foo|bar') + +# Detect python bug 1634343: "subprocess swallows empty arguments under win32" +# <http://sourceforge.net/tracker/index.php?func=detail&aid=1634343&group_id=5470&atid=105470> +# Also detect: "[ 1710802 ] subprocess must escape redirection characters under win32" +# <http://sourceforge.net/tracker/index.php?func=detail&aid=1710802&group_id=5470&atid=105470> +if has_subprocess_bug(): + import subprocess_fix + subprocess.list2cmdline = subprocess_fix.list2cmdline + +from sherrors import * + +class NonExitingParser(optparse.OptionParser): + """OptionParser default behaviour upon error is to print the error message and + exit. Raise a utility error instead. + """ + def error(self, msg): + raise UtilityError(msg) + +#------------------------------------------------------------------------------- +# set special builtin +#------------------------------------------------------------------------------- +OPT_SET = NonExitingParser(usage="set - set or unset options and positional parameters") +OPT_SET.add_option( '-f', action='store_true', dest='has_f', default=False, + help='The shell shall disable pathname expansion.') +OPT_SET.add_option('-e', action='store_true', dest='has_e', default=False, + help="""When this option is on, if a simple command fails for any of the \ + reasons listed in Consequences of Shell Errors or returns an exit status \ + value >0, and is not part of the compound list following a while, until, \ + or if keyword, and is not a part of an AND or OR list, and is not a \ + pipeline preceded by the ! reserved word, then the shell shall immediately \ + exit.""") +OPT_SET.add_option('-x', action='store_true', dest='has_x', default=False, + help="""The shell shall write to standard error a trace for each command \ + after it expands the command and before it executes it. It is unspecified \ + whether the command that turns tracing off is traced.""") + +def builtin_set(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_SET.parse_args(args) + env = interp.get_env() + + if option.has_f: + env.set_opt('-f') + if option.has_e: + env.set_opt('-e') + if option.has_x: + env.set_opt('-x') + return 0 + +#------------------------------------------------------------------------------- +# shift special builtin +#------------------------------------------------------------------------------- +def builtin_shift(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + params = interp.get_env().get_positional_args() + if args: + try: + n = int(args[0]) + if n > len(params): + raise ValueError() + except ValueError: + return 1 + else: + n = 1 + + params[:n] = [] + interp.get_env().set_positional_args(params) + return 0 + +#------------------------------------------------------------------------------- +# export special builtin +#------------------------------------------------------------------------------- +OPT_EXPORT = NonExitingParser(usage="set - set or unset options and positional parameters") +OPT_EXPORT.add_option('-p', action='store_true', dest='has_p', default=False) + +def builtin_export(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_EXPORT.parse_args(args) + if option.has_p: + raise NotImplementedError() + + for arg in args: + try: + name, value = arg.split('=', 1) + except ValueError: + name, value = arg, None + env = interp.get_env().export(name, value) + + return 0 + +#------------------------------------------------------------------------------- +# return special builtin +#------------------------------------------------------------------------------- +def builtin_return(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + res = 0 + if args: + try: + res = int(args[0]) + except ValueError: + res = 0 + if not 0<=res<=255: + res = 0 + + # BUG: should be last executed command exit code + raise ReturnSignal(res) + +#------------------------------------------------------------------------------- +# trap special builtin +#------------------------------------------------------------------------------- +def builtin_trap(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + if len(args) < 2: + stderr.write('trap: usage: trap [[arg] signal_spec ...]\n') + return 2 + + action = args[0] + for sig in args[1:]: + try: + env.traps[sig] = action + except Exception, e: + stderr.write('trap: %s\n' % str(e)) + return 0 + +#------------------------------------------------------------------------------- +# unset special builtin +#------------------------------------------------------------------------------- +OPT_UNSET = NonExitingParser("unset - unset values and attributes of variables and functions") +OPT_UNSET.add_option( '-f', action='store_true', dest='has_f', default=False) +OPT_UNSET.add_option( '-v', action='store_true', dest='has_v', default=False) + +def builtin_unset(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_UNSET.parse_args(args) + + status = 0 + env = interp.get_env() + for arg in args: + try: + if option.has_f: + env.remove_function(arg) + else: + del env[arg] + except KeyError: + pass + except VarAssignmentError: + status = 1 + + return status + +#------------------------------------------------------------------------------- +# wait special builtin +#------------------------------------------------------------------------------- +def builtin_wait(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + return interp.wait([int(arg) for arg in args]) + +#------------------------------------------------------------------------------- +# cat utility +#------------------------------------------------------------------------------- +def utility_cat(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + if not args: + args = ['-'] + + status = 0 + for arg in args: + if arg == '-': + data = stdin.read() + else: + path = os.path.join(env['PWD'], arg) + try: + f = file(path, 'rb') + try: + data = f.read() + finally: + f.close() + except IOError, e: + if e.errno != errno.ENOENT: + raise + status = 1 + continue + stdout.write(data) + stdout.flush() + return status + +#------------------------------------------------------------------------------- +# cd utility +#------------------------------------------------------------------------------- +OPT_CD = NonExitingParser("cd - change the working directory") + +def utility_cd(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_CD.parse_args(args) + env = interp.get_env() + + directory = None + printdir = False + if not args: + home = env.get('HOME') + if home: + # Unspecified, do nothing + return 0 + else: + directory = home + elif len(args)==1: + directory = args[0] + if directory=='-': + if 'OLDPWD' not in env: + raise UtilityError("OLDPWD not set") + printdir = True + directory = env['OLDPWD'] + else: + raise UtilityError("too many arguments") + + curpath = None + # Absolute directories will be handled correctly by the os.path.join call. + if not directory.startswith('.') and not directory.startswith('..'): + cdpaths = env.get('CDPATH', '.').split(';') + for cdpath in cdpaths: + p = os.path.join(cdpath, directory) + if os.path.isdir(p): + curpath = p + break + + if curpath is None: + curpath = directory + curpath = os.path.join(env['PWD'], directory) + + env['OLDPWD'] = env['PWD'] + env['PWD'] = curpath + if printdir: + stdout.write('%s\n' % curpath) + return 0 + +#------------------------------------------------------------------------------- +# colon utility +#------------------------------------------------------------------------------- +def utility_colon(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + return 0 + +#------------------------------------------------------------------------------- +# echo utility +#------------------------------------------------------------------------------- +def utility_echo(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + # Echo only takes arguments, no options. Use printf if you need fancy stuff. + output = ' '.join(args) + '\n' + stdout.write(output) + stdout.flush() + return 0 + +#------------------------------------------------------------------------------- +# egrep utility +#------------------------------------------------------------------------------- +# egrep is usually a shell script. +# Unfortunately, pysh does not support shell scripts *with arguments* right now, +# so the redirection is implemented here, assuming grep is available. +def utility_egrep(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + return run_command('grep', ['-E'] + args, interp, env, stdin, stdout, + stderr, debugflags) + +#------------------------------------------------------------------------------- +# env utility +#------------------------------------------------------------------------------- +def utility_env(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + if args and args[0]=='-i': + raise NotImplementedError('env: -i option is not implemented') + + i = 0 + for arg in args: + if '=' not in arg: + break + # Update the current environment + name, value = arg.split('=', 1) + env[name] = value + i += 1 + + if args[i:]: + # Find then execute the specified interpreter + utility = env.find_in_path(args[i]) + if not utility: + return 127 + args[i:i+1] = utility + name = args[i] + args = args[i+1:] + try: + return run_command(name, args, interp, env, stdin, stdout, stderr, + debugflags) + except UtilityError: + stderr.write('env: failed to execute %s' % ' '.join([name]+args)) + return 126 + else: + for pair in env.get_variables().iteritems(): + stdout.write('%s=%s\n' % pair) + return 0 + +#------------------------------------------------------------------------------- +# exit utility +#------------------------------------------------------------------------------- +def utility_exit(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + res = None + if args: + try: + res = int(args[0]) + except ValueError: + res = None + if not 0<=res<=255: + res = None + + if res is None: + # BUG: should be last executed command exit code + res = 0 + + raise ExitSignal(res) + +#------------------------------------------------------------------------------- +# fgrep utility +#------------------------------------------------------------------------------- +# see egrep +def utility_fgrep(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + return run_command('grep', ['-F'] + args, interp, env, stdin, stdout, + stderr, debugflags) + +#------------------------------------------------------------------------------- +# gunzip utility +#------------------------------------------------------------------------------- +# see egrep +def utility_gunzip(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + return run_command('gzip', ['-d'] + args, interp, env, stdin, stdout, + stderr, debugflags) + +#------------------------------------------------------------------------------- +# kill utility +#------------------------------------------------------------------------------- +def utility_kill(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + for arg in args: + pid = int(arg) + status = subprocess.call(['pskill', '/T', str(pid)], + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # pskill is asynchronous, hence the stupid polling loop + while 1: + p = subprocess.Popen(['pslist', str(pid)], + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = p.communicate()[0] + if ('process %d was not' % pid) in output: + break + time.sleep(1) + return status + +#------------------------------------------------------------------------------- +# mkdir utility +#------------------------------------------------------------------------------- +OPT_MKDIR = NonExitingParser("mkdir - make directories.") +OPT_MKDIR.add_option('-p', action='store_true', dest='has_p', default=False) + +def utility_mkdir(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + # TODO: implement umask + # TODO: implement proper utility error report + option, args = OPT_MKDIR.parse_args(args) + for arg in args: + path = os.path.join(env['PWD'], arg) + if option.has_p: + try: + os.makedirs(path) + except IOError, e: + if e.errno != errno.EEXIST: + raise + else: + os.mkdir(path) + return 0 + +#------------------------------------------------------------------------------- +# netstat utility +#------------------------------------------------------------------------------- +def utility_netstat(name, args, interp, env, stdin, stdout, stderr, debugflags): + # Do you really expect me to implement netstat ? + # This empty form is enough for Mercurial tests since it's + # supposed to generate nothing upon success. Faking this test + # is not a big deal either. + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + return 0 + +#------------------------------------------------------------------------------- +# pwd utility +#------------------------------------------------------------------------------- +OPT_PWD = NonExitingParser("pwd - return working directory name") +OPT_PWD.add_option('-L', action='store_true', dest='has_L', default=True, + help="""If the PWD environment variable contains an absolute pathname of \ + the current directory that does not contain the filenames dot or dot-dot, \ + pwd shall write this pathname to standard output. Otherwise, the -L option \ + shall behave as the -P option.""") +OPT_PWD.add_option('-P', action='store_true', dest='has_L', default=False, + help="""The absolute pathname written shall not contain filenames that, in \ + the context of the pathname, refer to files of type symbolic link.""") + +def utility_pwd(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_PWD.parse_args(args) + stdout.write('%s\n' % env['PWD']) + return 0 + +#------------------------------------------------------------------------------- +# printf utility +#------------------------------------------------------------------------------- +RE_UNESCAPE = re.compile(r'(\\x[a-zA-Z0-9]{2}|\\[0-7]{1,3}|\\.)') + +def utility_printf(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + def replace(m): + assert m.group() + g = m.group()[1:] + if g.startswith('x'): + return chr(int(g[1:], 16)) + if len(g) <= 3 and len([c for c in g if c in '01234567']) == len(g): + # Yay, an octal number + return chr(int(g, 8)) + return { + 'a': '\a', + 'b': '\b', + 'f': '\f', + 'n': '\n', + 'r': '\r', + 't': '\t', + 'v': '\v', + '\\': '\\', + }.get(g) + + # Convert escape sequences + format = re.sub(RE_UNESCAPE, replace, args[0]) + stdout.write(format % tuple(args[1:])) + return 0 + +#------------------------------------------------------------------------------- +# true utility +#------------------------------------------------------------------------------- +def utility_true(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + return 0 + +#------------------------------------------------------------------------------- +# sed utility +#------------------------------------------------------------------------------- +RE_SED = re.compile(r'^s(.).*\1[a-zA-Z]*$') + +# cygwin sed fails with some expressions when they do not end with a single space. +# see unit tests for details. Interestingly, the same expressions works perfectly +# in cygwin shell. +def utility_sed(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + # Scan pattern arguments and append a space if necessary + for i in xrange(len(args)): + if not RE_SED.search(args[i]): + continue + args[i] = args[i] + ' ' + + return run_command(name, args, interp, env, stdin, stdout, + stderr, debugflags) + +#------------------------------------------------------------------------------- +# sleep utility +#------------------------------------------------------------------------------- +def utility_sleep(name, args, interp, env, stdin, stdout, stderr, debugflags): + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + time.sleep(int(args[0])) + return 0 + +#------------------------------------------------------------------------------- +# sort utility +#------------------------------------------------------------------------------- +OPT_SORT = NonExitingParser("sort - sort, merge, or sequence check text files") + +def utility_sort(name, args, interp, env, stdin, stdout, stderr, debugflags): + + def sort(path): + if path == '-': + lines = stdin.readlines() + else: + try: + f = file(path) + try: + lines = f.readlines() + finally: + f.close() + except IOError, e: + stderr.write(str(e) + '\n') + return 1 + + if lines and lines[-1][-1]!='\n': + lines[-1] = lines[-1] + '\n' + return lines + + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + option, args = OPT_SORT.parse_args(args) + alllines = [] + + if len(args)<=0: + args += ['-'] + + # Load all files lines + curdir = os.getcwd() + try: + os.chdir(env['PWD']) + for path in args: + alllines += sort(path) + finally: + os.chdir(curdir) + + alllines.sort() + for line in alllines: + stdout.write(line) + return 0 + +#------------------------------------------------------------------------------- +# hg utility +#------------------------------------------------------------------------------- + +hgcommands = [ + 'add', + 'addremove', + 'commit', 'ci', + 'debugrename', + 'debugwalk', + 'falabala', # Dummy command used in a mercurial test + 'incoming', + 'locate', + 'pull', + 'push', + 'qinit', + 'remove', 'rm', + 'rename', 'mv', + 'revert', + 'showconfig', + 'status', 'st', + 'strip', + ] + +def rewriteslashes(name, args): + # Several hg commands output file paths, rewrite the separators + if len(args) > 1 and name.lower().endswith('python') \ + and args[0].endswith('hg'): + for cmd in hgcommands: + if cmd in args[1:]: + return True + + # svn output contains many paths with OS specific separators. + # Normalize these to unix paths. + base = os.path.basename(name) + if base.startswith('svn'): + return True + + return False + +def rewritehg(output): + if not output: + return output + # Rewrite os specific messages + output = output.replace(': The system cannot find the file specified', + ': No such file or directory') + output = re.sub(': Access is denied.*$', ': Permission denied', output) + output = output.replace(': No connection could be made because the target machine actively refused it', + ': Connection refused') + return output + + +def run_command(name, args, interp, env, stdin, stdout, + stderr, debugflags): + # Execute the command + if 'debug-utility' in debugflags: + print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') + + hgbin = interp.options().hgbinary + ishg = hgbin and ('hg' in name or args and 'hg' in args[0]) + unixoutput = 'cygwin' in name or ishg + + exec_env = env.get_variables() + try: + # BUG: comparing file descriptor is clearly not a reliable way to tell + # whether they point on the same underlying object. But in pysh limited + # scope this is usually right, we do not expect complicated redirections + # besides usual 2>&1. + # Still there is one case we have but cannot deal with is when stdout + # and stderr are redirected *by pysh caller*. This the reason for the + # --redirect pysh() option. + # Now, we want to know they are the same because we sometimes need to + # transform the command output, mostly remove CR-LF to ensure that + # command output is unix-like. Cygwin utilies are a special case because + # they explicitely set their output streams to binary mode, so we have + # nothing to do. For all others commands, we have to guess whether they + # are sending text data, in which case the transformation must be done. + # Again, the NUL character test is unreliable but should be enough for + # hg tests. + redirected = stdout.fileno()==stderr.fileno() + if not redirected: + p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env, + stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env, + stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + out, err = p.communicate() + except WindowsError, e: + raise UtilityError(str(e)) + + if not unixoutput: + def encode(s): + if '\0' in s: + return s + return s.replace('\r\n', '\n') + else: + encode = lambda s: s + + if rewriteslashes(name, args): + encode1_ = encode + def encode(s): + s = encode1_(s) + s = s.replace('\\\\', '\\') + s = s.replace('\\', '/') + return s + + if ishg: + encode2_ = encode + def encode(s): + return rewritehg(encode2_(s)) + + stdout.write(encode(out)) + if not redirected: + stderr.write(encode(err)) + return p.returncode + diff --git a/bitbake/lib/pysh/interp.py b/bitbake/lib/pysh/interp.py new file mode 100644 index 0000000000..efe5181e1e --- /dev/null +++ b/bitbake/lib/pysh/interp.py @@ -0,0 +1,1367 @@ +# interp.py - shell interpreter for pysh. +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +"""Implement the shell interpreter. + +Most references are made to "The Open Group Base Specifications Issue 6". +<http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html> +""" +# TODO: document the fact input streams must implement fileno() so Popen will work correctly. +# it requires non-stdin stream to be implemented as files. Still to be tested... +# DOC: pathsep is used in PATH instead of ':'. Clearly, there are path syntax issues here. +# TODO: stop command execution upon error. +# TODO: sort out the filename/io_number mess. It should be possible to use filenames only. +# TODO: review subshell implementation +# TODO: test environment cloning for non-special builtins +# TODO: set -x should not rebuild commands from tokens, assignments/redirections are lost +# TODO: unit test for variable assignment +# TODO: test error management wrt error type/utility type +# TODO: test for binary output everywhere +# BUG: debug-parsing does not pass log file to PLY. Maybe a PLY upgrade is necessary. +import base64 +import cPickle as pickle +import errno +import glob +import os +import re +import subprocess +import sys +import tempfile + +try: + s = set() + del s +except NameError: + from Set import Set as set + +import builtin +from sherrors import * +import pyshlex +import pyshyacc + +def mappend(func, *args, **kargs): + """Like map but assume func returns a list. Returned lists are merged into + a single one. + """ + return reduce(lambda a,b: a+b, map(func, *args, **kargs), []) + +class FileWrapper: + """File object wrapper to ease debugging. + + Allow mode checking and implement file duplication through a simple + reference counting scheme. Not sure the latter is really useful since + only real file descriptors can be used. + """ + def __init__(self, mode, file, close=True): + if mode not in ('r', 'w', 'a'): + raise IOError('invalid mode: %s' % mode) + self._mode = mode + self._close = close + if isinstance(file, FileWrapper): + if file._refcount[0] <= 0: + raise IOError(0, 'Error') + self._refcount = file._refcount + self._refcount[0] += 1 + self._file = file._file + else: + self._refcount = [1] + self._file = file + + def dup(self): + return FileWrapper(self._mode, self, self._close) + + def fileno(self): + """fileno() should be only necessary for input streams.""" + return self._file.fileno() + + def read(self, size=-1): + if self._mode!='r': + raise IOError(0, 'Error') + return self._file.read(size) + + def readlines(self, *args, **kwargs): + return self._file.readlines(*args, **kwargs) + + def write(self, s): + if self._mode not in ('w', 'a'): + raise IOError(0, 'Error') + return self._file.write(s) + + def flush(self): + self._file.flush() + + def close(self): + if not self._refcount: + return + assert self._refcount[0] > 0 + + self._refcount[0] -= 1 + if self._refcount[0] == 0: + self._mode = 'c' + if self._close: + self._file.close() + self._refcount = None + + def mode(self): + return self._mode + + def __getattr__(self, name): + if name == 'name': + self.name = getattr(self._file, name) + return self.name + else: + raise AttributeError(name) + + def __del__(self): + self.close() + + +def win32_open_devnull(mode): + return open('NUL', mode) + + +class Redirections: + """Stores open files and their mapping to pseudo-sh file descriptor. + """ + # BUG: redirections are not handled correctly: 1>&3 2>&3 3>&4 does + # not make 1 to redirect to 4 + def __init__(self, stdin=None, stdout=None, stderr=None): + self._descriptors = {} + if stdin is not None: + self._add_descriptor(0, stdin) + if stdout is not None: + self._add_descriptor(1, stdout) + if stderr is not None: + self._add_descriptor(2, stderr) + + def add_here_document(self, interp, name, content, io_number=None): + if io_number is None: + io_number = 0 + + if name==pyshlex.unquote_wordtree(name): + content = interp.expand_here_document(('TOKEN', content)) + + # Write document content in a temporary file + tmp = tempfile.TemporaryFile() + try: + tmp.write(content) + tmp.flush() + tmp.seek(0) + self._add_descriptor(io_number, FileWrapper('r', tmp)) + except: + tmp.close() + raise + + def add(self, interp, op, filename, io_number=None): + if op not in ('<', '>', '>|', '>>', '>&'): + # TODO: add descriptor duplication and here_documents + raise RedirectionError('Unsupported redirection operator "%s"' % op) + + if io_number is not None: + io_number = int(io_number) + + if (op == '>&' and filename.isdigit()) or filename=='-': + # No expansion for file descriptors, quote them if you want a filename + fullname = filename + else: + if filename.startswith('/'): + # TODO: win32 kludge + if filename=='/dev/null': + fullname = 'NUL' + else: + # TODO: handle absolute pathnames, they are unlikely to exist on the + # current platform (win32 for instance). + raise NotImplementedError() + else: + fullname = interp.expand_redirection(('TOKEN', filename)) + if not fullname: + raise RedirectionError('%s: ambiguous redirect' % filename) + # Build absolute path based on PWD + fullname = os.path.join(interp.get_env()['PWD'], fullname) + + if op=='<': + return self._add_input_redirection(interp, fullname, io_number) + elif op in ('>', '>|'): + clobber = ('>|'==op) + return self._add_output_redirection(interp, fullname, io_number, clobber) + elif op=='>>': + return self._add_output_appending(interp, fullname, io_number) + elif op=='>&': + return self._dup_output_descriptor(fullname, io_number) + + def close(self): + if self._descriptors is not None: + for desc in self._descriptors.itervalues(): + desc.flush() + desc.close() + self._descriptors = None + + def stdin(self): + return self._descriptors[0] + + def stdout(self): + return self._descriptors[1] + + def stderr(self): + return self._descriptors[2] + + def clone(self): + clone = Redirections() + for desc, fileobj in self._descriptors.iteritems(): + clone._descriptors[desc] = fileobj.dup() + return clone + + def _add_output_redirection(self, interp, filename, io_number, clobber): + if io_number is None: + # io_number default to standard output + io_number = 1 + + if not clobber and interp.get_env().has_opt('-C') and os.path.isfile(filename): + # File already exist in no-clobber mode, bail out + raise RedirectionError('File "%s" already exists' % filename) + + # Open and register + self._add_file_descriptor(io_number, filename, 'w') + + def _add_output_appending(self, interp, filename, io_number): + if io_number is None: + io_number = 1 + self._add_file_descriptor(io_number, filename, 'a') + + def _add_input_redirection(self, interp, filename, io_number): + if io_number is None: + io_number = 0 + self._add_file_descriptor(io_number, filename, 'r') + + def _add_file_descriptor(self, io_number, filename, mode): + try: + if filename.startswith('/'): + if filename=='/dev/null': + f = win32_open_devnull(mode+'b') + else: + # TODO: handle absolute pathnames, they are unlikely to exist on the + # current platform (win32 for instance). + raise NotImplementedError('cannot open absolute path %s' % repr(filename)) + else: + f = file(filename, mode+'b') + except IOError, e: + raise RedirectionError(str(e)) + + wrapper = None + try: + wrapper = FileWrapper(mode, f) + f = None + self._add_descriptor(io_number, wrapper) + except: + if f: f.close() + if wrapper: wrapper.close() + raise + + def _dup_output_descriptor(self, source_fd, dest_fd): + if source_fd is None: + source_fd = 1 + self._dup_file_descriptor(source_fd, dest_fd, 'w') + + def _dup_file_descriptor(self, source_fd, dest_fd, mode): + source_fd = int(source_fd) + if source_fd not in self._descriptors: + raise RedirectionError('"%s" is not a valid file descriptor' % str(source_fd)) + source = self._descriptors[source_fd] + + if source.mode()!=mode: + raise RedirectionError('Descriptor %s cannot be duplicated in mode "%s"' % (str(source), mode)) + + if dest_fd=='-': + # Close the source descriptor + del self._descriptors[source_fd] + source.close() + else: + dest_fd = int(dest_fd) + if dest_fd not in self._descriptors: + raise RedirectionError('Cannot replace file descriptor %s' % str(dest_fd)) + + dest = self._descriptors[dest_fd] + if dest.mode()!=mode: + raise RedirectionError('Descriptor %s cannot be cannot be redirected in mode "%s"' % (str(dest), mode)) + + self._descriptors[dest_fd] = source.dup() + dest.close() + + def _add_descriptor(self, io_number, file): + io_number = int(io_number) + + if io_number in self._descriptors: + # Close the current descriptor + d = self._descriptors[io_number] + del self._descriptors[io_number] + d.close() + + self._descriptors[io_number] = file + + def __str__(self): + names = [('%d=%r' % (k, getattr(v, 'name', None))) for k,v + in self._descriptors.iteritems()] + names = ','.join(names) + return 'Redirections(%s)' % names + + def __del__(self): + self.close() + +def cygwin_to_windows_path(path): + """Turn /cygdrive/c/foo into c:/foo, or return path if it + is not a cygwin path. + """ + if not path.startswith('/cygdrive/'): + return path + path = path[len('/cygdrive/'):] + path = path[:1] + ':' + path[1:] + return path + +def win32_to_unix_path(path): + if path is not None: + path = path.replace('\\', '/') + return path + +_RE_SHEBANG = re.compile(r'^\#!\s?([^\s]+)(?:\s([^\s]+))?') +_SHEBANG_CMDS = { + '/usr/bin/env': 'env', + '/bin/sh': 'pysh', + 'python': 'python', +} + +def resolve_shebang(path, ignoreshell=False): + """Return a list of arguments as shebang interpreter call or an empty list + if path does not refer to an executable script. + See <http://www.opengroup.org/austin/docs/austin_51r2.txt>. + + ignoreshell - set to True to ignore sh shebangs. Return an empty list instead. + """ + try: + f = file(path) + try: + # At most 80 characters in the first line + header = f.read(80).splitlines()[0] + finally: + f.close() + + m = _RE_SHEBANG.search(header) + if not m: + return [] + cmd, arg = m.group(1,2) + if os.path.isfile(cmd): + # Keep this one, the hg script for instance contains a weird windows + # shebang referencing the current python install. + cmdfile = os.path.basename(cmd).lower() + if cmdfile == 'python.exe': + cmd = 'python' + pass + elif cmd not in _SHEBANG_CMDS: + raise CommandNotFound('Unknown interpreter "%s" referenced in '\ + 'shebang' % header) + cmd = _SHEBANG_CMDS.get(cmd) + if cmd is None or (ignoreshell and cmd == 'pysh'): + return [] + if arg is None: + return [cmd, win32_to_unix_path(path)] + return [cmd, arg, win32_to_unix_path(path)] + except IOError, e: + if e.errno!=errno.ENOENT and \ + (e.errno!=errno.EPERM and not os.path.isdir(path)): # Opening a directory raises EPERM + raise + return [] + +def win32_find_in_path(name, path): + if isinstance(path, str): + path = path.split(os.pathsep) + + exts = os.environ.get('PATHEXT', '').lower().split(os.pathsep) + for p in path: + p_name = os.path.join(p, name) + + prefix = resolve_shebang(p_name) + if prefix: + return prefix + + for ext in exts: + p_name_ext = p_name + ext + if os.path.exists(p_name_ext): + return [win32_to_unix_path(p_name_ext)] + return [] + +class Traps(dict): + def __setitem__(self, key, value): + if key not in ('EXIT',): + raise NotImplementedError() + super(Traps, self).__setitem__(key, value) + +# IFS white spaces character class +_IFS_WHITESPACES = (' ', '\t', '\n') + +class Environment: + """Environment holds environment variables, export table, function + definitions and whatever is defined in 2.12 "Shell Execution Environment", + redirection excepted. + """ + def __init__(self, pwd): + self._opt = set() #Shell options + + self._functions = {} + self._env = {'?': '0', '#': '0'} + self._exported = set([ + 'HOME', 'IFS', 'PATH' + ]) + + # Set environment vars with side-effects + self._ifs_ws = None # Set of IFS whitespace characters + self._ifs_re = None # Regular expression used to split between words using IFS classes + self['IFS'] = ''.join(_IFS_WHITESPACES) #Default environment values + self['PWD'] = pwd + self.traps = Traps() + + def clone(self, subshell=False): + env = Environment(self['PWD']) + env._opt = set(self._opt) + for k,v in self.get_variables().iteritems(): + if k in self._exported: + env.export(k,v) + elif subshell: + env[k] = v + + if subshell: + env._functions = dict(self._functions) + + return env + + def __getitem__(self, key): + if key in ('@', '*', '-', '$'): + raise NotImplementedError('%s is not implemented' % repr(key)) + return self._env[key] + + def get(self, key, defval=None): + try: + return self[key] + except KeyError: + return defval + + def __setitem__(self, key, value): + if key=='IFS': + # Update the whitespace/non-whitespace classes + self._update_ifs(value) + elif key=='PWD': + pwd = os.path.abspath(value) + if not os.path.isdir(pwd): + raise VarAssignmentError('Invalid directory %s' % value) + value = pwd + elif key in ('?', '!'): + value = str(int(value)) + self._env[key] = value + + def __delitem__(self, key): + if key in ('IFS', 'PWD', '?'): + raise VarAssignmentError('%s cannot be unset' % key) + del self._env[key] + + def __contains__(self, item): + return item in self._env + + def set_positional_args(self, args): + """Set the content of 'args' as positional argument from 1 to len(args). + Return previous argument as a list of strings. + """ + # Save and remove previous arguments + prevargs = [] + for i in xrange(int(self._env['#'])): + i = str(i+1) + prevargs.append(self._env[i]) + del self._env[i] + self._env['#'] = '0' + + #Set new ones + for i,arg in enumerate(args): + self._env[str(i+1)] = str(arg) + self._env['#'] = str(len(args)) + + return prevargs + + def get_positional_args(self): + return [self._env[str(i+1)] for i in xrange(int(self._env['#']))] + + def get_variables(self): + return dict(self._env) + + def export(self, key, value=None): + if value is not None: + self[key] = value + self._exported.add(key) + + def get_exported(self): + return [(k,self._env.get(k)) for k in self._exported] + + def split_fields(self, word): + if not self._ifs_ws or not word: + return [word] + return re.split(self._ifs_re, word) + + def _update_ifs(self, value): + """Update the split_fields related variables when IFS character set is + changed. + """ + # TODO: handle NULL IFS + + # Separate characters in whitespace and non-whitespace + chars = set(value) + ws = [c for c in chars if c in _IFS_WHITESPACES] + nws = [c for c in chars if c not in _IFS_WHITESPACES] + + # Keep whitespaces in a string for left and right stripping + self._ifs_ws = ''.join(ws) + + # Build a regexp to split fields + trailing = '[' + ''.join([re.escape(c) for c in ws]) + ']' + if nws: + # First, the single non-whitespace occurence. + nws = '[' + ''.join([re.escape(c) for c in nws]) + ']' + nws = '(?:' + trailing + '*' + nws + trailing + '*' + '|' + trailing + '+)' + else: + # Then mix all parts with quantifiers + nws = trailing + '+' + self._ifs_re = re.compile(nws) + + def has_opt(self, opt, val=None): + return (opt, val) in self._opt + + def set_opt(self, opt, val=None): + self._opt.add((opt, val)) + + def find_in_path(self, name, pwd=False): + path = self._env.get('PATH', '').split(os.pathsep) + if pwd: + path[:0] = [self['PWD']] + if os.name == 'nt': + return win32_find_in_path(name, self._env.get('PATH', '')) + else: + raise NotImplementedError() + + def define_function(self, name, body): + if not is_name(name): + raise ShellSyntaxError('%s is not a valid function name' % repr(name)) + self._functions[name] = body + + def remove_function(self, name): + del self._functions[name] + + def is_function(self, name): + return name in self._functions + + def get_function(self, name): + return self._functions.get(name) + + +name_charset = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_' +name_charset = dict(zip(name_charset,name_charset)) + +def match_name(s): + """Return the length in characters of the longest prefix made of name + allowed characters in s. + """ + for i,c in enumerate(s): + if c not in name_charset: + return s[:i] + return s + +def is_name(s): + return len([c for c in s if c not in name_charset])<=0 + +def is_special_param(c): + return len(c)==1 and c in ('@','*','#','?','-','$','!','0') + +def utility_not_implemented(name, *args, **kwargs): + raise NotImplementedError('%s utility is not implemented' % name) + + +class Utility: + """Define utilities properties: + func -- utility callable. See builtin module for utility samples. + is_special -- see XCU 2.8. + """ + def __init__(self, func, is_special=0): + self.func = func + self.is_special = bool(is_special) + + +def encodeargs(args): + def encodearg(s): + lines = base64.encodestring(s) + lines = [l.splitlines()[0] for l in lines] + return ''.join(lines) + + s = pickle.dumps(args) + return encodearg(s) + +def decodeargs(s): + s = base64.decodestring(s) + return pickle.loads(s) + + +class GlobError(Exception): + pass + +class Options: + def __init__(self): + # True if Mercurial operates with binary streams + self.hgbinary = True + +class Interpreter: + # Implementation is very basic: the execute() method just makes a DFS on the + # AST and execute nodes one by one. Nodes are tuple (name,obj) where name + # is a string identifier and obj the AST element returned by the parser. + # + # Handler are named after the node identifiers. + # TODO: check node names and remove the switch in execute with some + # dynamic getattr() call to find node handlers. + """Shell interpreter. + + The following debugging flags can be passed: + debug-parsing - enable PLY debugging. + debug-tree - print the generated AST. + debug-cmd - trace command execution before word expansion, plus exit status. + debug-utility - trace utility execution. + """ + + # List supported commands. + COMMANDS = { + 'cat': Utility(builtin.utility_cat,), + 'cd': Utility(builtin.utility_cd,), + ':': Utility(builtin.utility_colon,), + 'echo': Utility(builtin.utility_echo), + 'env': Utility(builtin.utility_env), + 'exit': Utility(builtin.utility_exit), + 'export': Utility(builtin.builtin_export, is_special=1), + 'egrep': Utility(builtin.utility_egrep), + 'fgrep': Utility(builtin.utility_fgrep), + 'gunzip': Utility(builtin.utility_gunzip), + 'kill': Utility(builtin.utility_kill), + 'mkdir': Utility(builtin.utility_mkdir), + 'netstat': Utility(builtin.utility_netstat), + 'printf': Utility(builtin.utility_printf), + 'pwd': Utility(builtin.utility_pwd), + 'return': Utility(builtin.builtin_return, is_special=1), + 'sed': Utility(builtin.utility_sed,), + 'set': Utility(builtin.builtin_set,), + 'shift': Utility(builtin.builtin_shift,), + 'sleep': Utility(builtin.utility_sleep,), + 'sort': Utility(builtin.utility_sort,), + 'trap': Utility(builtin.builtin_trap, is_special=1), + 'true': Utility(builtin.utility_true), + 'unset': Utility(builtin.builtin_unset, is_special=1), + 'wait': Utility(builtin.builtin_wait, is_special=1), + } + + def __init__(self, pwd, debugflags = [], env=None, redirs=None, stdin=None, + stdout=None, stderr=None, opts=Options()): + self._env = env + if self._env is None: + self._env = Environment(pwd) + self._children = {} + + self._redirs = redirs + self._close_redirs = False + + if self._redirs is None: + if stdin is None: + stdin = sys.stdin + if stdout is None: + stdout = sys.stdout + if stderr is None: + stderr = sys.stderr + stdin = FileWrapper('r', stdin, False) + stdout = FileWrapper('w', stdout, False) + stderr = FileWrapper('w', stderr, False) + self._redirs = Redirections(stdin, stdout, stderr) + self._close_redirs = True + + self._debugflags = list(debugflags) + self._logfile = sys.stderr + self._options = opts + + def close(self): + """Must be called when the interpreter is no longer used.""" + script = self._env.traps.get('EXIT') + if script: + try: + self.execute_script(script=script) + except: + pass + + if self._redirs is not None and self._close_redirs: + self._redirs.close() + self._redirs = None + + def log(self, s): + self._logfile.write(s) + self._logfile.flush() + + def __getitem__(self, key): + return self._env[key] + + def __setitem__(self, key, value): + self._env[key] = value + + def options(self): + return self._options + + def redirect(self, redirs, ios): + def add_redir(io): + if isinstance(io, pyshyacc.IORedirect): + redirs.add(self, io.op, io.filename, io.io_number) + else: + redirs.add_here_document(self, io.name, io.content, io.io_number) + + map(add_redir, ios) + return redirs + + def execute_script(self, script=None, ast=None, sourced=False, + scriptpath=None): + """If script is not None, parse the input. Otherwise takes the supplied + AST. Then execute the AST. + Return the script exit status. + """ + try: + if scriptpath is not None: + self._env['0'] = os.path.abspath(scriptpath) + + if script is not None: + debug_parsing = ('debug-parsing' in self._debugflags) + cmds, script = pyshyacc.parse(script, True, debug_parsing) + if 'debug-tree' in self._debugflags: + pyshyacc.print_commands(cmds, self._logfile) + self._logfile.flush() + else: + cmds, script = ast, '' + + status = 0 + for cmd in cmds: + try: + status = self.execute(cmd) + except ExitSignal, e: + if sourced: + raise + status = int(e.args[0]) + return status + except ShellError: + self._env['?'] = 1 + raise + if 'debug-utility' in self._debugflags or 'debug-cmd' in self._debugflags: + self.log('returncode ' + str(status)+ '\n') + return status + except CommandNotFound, e: + print >>self._redirs.stderr, str(e) + self._redirs.stderr.flush() + # Command not found by non-interactive shell + # return 127 + raise + except RedirectionError, e: + # TODO: should be handled depending on the utility status + print >>self._redirs.stderr, str(e) + self._redirs.stderr.flush() + # Command not found by non-interactive shell + # return 127 + raise + + def dotcommand(self, env, args): + if len(args) < 1: + raise ShellError('. expects at least one argument') + path = args[0] + if '/' not in path: + found = env.find_in_path(args[0], True) + if found: + path = found[0] + script = file(path).read() + return self.execute_script(script=script, sourced=True) + + def execute(self, token, redirs=None): + """Execute and AST subtree with supplied redirections overriding default + interpreter ones. + Return the exit status. + """ + if not token: + return 0 + + if redirs is None: + redirs = self._redirs + + if isinstance(token, list): + # Commands sequence + res = 0 + for t in token: + res = self.execute(t, redirs) + return res + + type, value = token + status = 0 + if type=='simple_command': + redirs_copy = redirs.clone() + try: + # TODO: define and handle command return values + # TODO: implement set -e + status = self._execute_simple_command(value, redirs_copy) + finally: + redirs_copy.close() + elif type=='pipeline': + status = self._execute_pipeline(value, redirs) + elif type=='and_or': + status = self._execute_and_or(value, redirs) + elif type=='for_clause': + status = self._execute_for_clause(value, redirs) + elif type=='while_clause': + status = self._execute_while_clause(value, redirs) + elif type=='function_definition': + status = self._execute_function_definition(value, redirs) + elif type=='brace_group': + status = self._execute_brace_group(value, redirs) + elif type=='if_clause': + status = self._execute_if_clause(value, redirs) + elif type=='subshell': + status = self.subshell(ast=value.cmds, redirs=redirs) + elif type=='async': + status = self._asynclist(value) + elif type=='redirect_list': + redirs_copy = self.redirect(redirs.clone(), value.redirs) + try: + status = self.execute(value.cmd, redirs_copy) + finally: + redirs_copy.close() + else: + raise NotImplementedError('Unsupported token type ' + type) + + if status < 0: + status = 255 + return status + + def _execute_if_clause(self, if_clause, redirs): + cond_status = self.execute(if_clause.cond, redirs) + if cond_status==0: + return self.execute(if_clause.if_cmds, redirs) + else: + return self.execute(if_clause.else_cmds, redirs) + + def _execute_brace_group(self, group, redirs): + status = 0 + for cmd in group.cmds: + status = self.execute(cmd, redirs) + return status + + def _execute_function_definition(self, fundef, redirs): + self._env.define_function(fundef.name, fundef.body) + return 0 + + def _execute_while_clause(self, while_clause, redirs): + status = 0 + while 1: + cond_status = 0 + for cond in while_clause.condition: + cond_status = self.execute(cond, redirs) + + if cond_status: + break + + for cmd in while_clause.cmds: + status = self.execute(cmd, redirs) + + return status + + def _execute_for_clause(self, for_clause, redirs): + if not is_name(for_clause.name): + raise ShellSyntaxError('%s is not a valid name' % repr(for_clause.name)) + items = mappend(self.expand_token, for_clause.items) + + status = 0 + for item in items: + self._env[for_clause.name] = item + for cmd in for_clause.cmds: + status = self.execute(cmd, redirs) + return status + + def _execute_and_or(self, or_and, redirs): + res = self.execute(or_and.left, redirs) + if (or_and.op=='&&' and res==0) or (or_and.op!='&&' and res!=0): + res = self.execute(or_and.right, redirs) + return res + + def _execute_pipeline(self, pipeline, redirs): + if len(pipeline.commands)==1: + status = self.execute(pipeline.commands[0], redirs) + else: + # Execute all commands one after the other + status = 0 + inpath, outpath = None, None + try: + # Commands inputs and outputs cannot really be plugged as done + # by a real shell. Run commands sequentially and chain their + # input/output throught temporary files. + tmpfd, inpath = tempfile.mkstemp() + os.close(tmpfd) + tmpfd, outpath = tempfile.mkstemp() + os.close(tmpfd) + + inpath = win32_to_unix_path(inpath) + outpath = win32_to_unix_path(outpath) + + for i, cmd in enumerate(pipeline.commands): + call_redirs = redirs.clone() + try: + if i!=0: + call_redirs.add(self, '<', inpath) + if i!=len(pipeline.commands)-1: + call_redirs.add(self, '>', outpath) + + status = self.execute(cmd, call_redirs) + + # Chain inputs/outputs + inpath, outpath = outpath, inpath + finally: + call_redirs.close() + finally: + if inpath: os.remove(inpath) + if outpath: os.remove(outpath) + + if pipeline.reverse_status: + status = int(not status) + self._env['?'] = status + return status + + def _execute_function(self, name, args, interp, env, stdin, stdout, stderr, *others): + assert interp is self + + func = env.get_function(name) + #Set positional parameters + prevargs = None + try: + prevargs = env.set_positional_args(args) + try: + redirs = Redirections(stdin.dup(), stdout.dup(), stderr.dup()) + try: + status = self.execute(func, redirs) + finally: + redirs.close() + except ReturnSignal, e: + status = int(e.args[0]) + env['?'] = status + return status + finally: + #Reset positional parameters + if prevargs is not None: + env.set_positional_args(prevargs) + + def _execute_simple_command(self, token, redirs): + """Can raise ReturnSignal when return builtin is called, ExitSignal when + exit is called, and other shell exceptions upon builtin failures. + """ + debug_command = 'debug-cmd' in self._debugflags + if debug_command: + self.log('word' + repr(token.words) + '\n') + self.log('assigns' + repr(token.assigns) + '\n') + self.log('redirs' + repr(token.redirs) + '\n') + + is_special = None + env = self._env + + try: + # Word expansion + args = [] + for word in token.words: + args += self.expand_token(word) + if is_special is None and args: + is_special = env.is_function(args[0]) or \ + (args[0] in self.COMMANDS and self.COMMANDS[args[0]].is_special) + + if debug_command: + self.log('_execute_simple_command' + str(args) + '\n') + + if not args: + # Redirections happen is a subshell + redirs = redirs.clone() + elif not is_special: + env = self._env.clone() + + # Redirections + self.redirect(redirs, token.redirs) + + # Variables assignments + res = 0 + for type,(k,v) in token.assigns: + status, expanded = self.expand_variable((k,v)) + if status is not None: + res = status + if args: + env.export(k, expanded) + else: + env[k] = expanded + + if args and args[0] in ('.', 'source'): + res = self.dotcommand(env, args[1:]) + elif args: + if args[0] in self.COMMANDS: + command = self.COMMANDS[args[0]] + elif env.is_function(args[0]): + command = Utility(self._execute_function, is_special=True) + else: + if not '/' in args[0].replace('\\', '/'): + cmd = env.find_in_path(args[0]) + if not cmd: + # TODO: test error code on unknown command => 127 + raise CommandNotFound('Unknown command: "%s"' % args[0]) + else: + # Handle commands like '/cygdrive/c/foo.bat' + cmd = cygwin_to_windows_path(args[0]) + if not os.path.exists(cmd): + raise CommandNotFound('%s: No such file or directory' % args[0]) + shebang = resolve_shebang(cmd) + if shebang: + cmd = shebang + else: + cmd = [cmd] + args[0:1] = cmd + command = Utility(builtin.run_command) + + # Command execution + if 'debug-cmd' in self._debugflags: + self.log('redirections ' + str(redirs) + '\n') + + res = command.func(args[0], args[1:], self, env, + redirs.stdin(), redirs.stdout(), + redirs.stderr(), self._debugflags) + + if self._env.has_opt('-x'): + # Trace command execution in shell environment + # BUG: would be hard to reproduce a real shell behaviour since + # the AST is not annotated with source lines/tokens. + self._redirs.stdout().write(' '.join(args)) + + except ReturnSignal: + raise + except ShellError, e: + if is_special or isinstance(e, (ExitSignal, + ShellSyntaxError, ExpansionError)): + raise e + self._redirs.stderr().write(str(e)+'\n') + return 1 + + return res + + def expand_token(self, word): + """Expand a word as specified in [2.6 Word Expansions]. Return the list + of expanded words. + """ + status, wtrees = self._expand_word(word) + return map(pyshlex.wordtree_as_string, wtrees) + + def expand_variable(self, word): + """Return a status code (or None if no command expansion occurred) + and a single word. + """ + status, wtrees = self._expand_word(word, pathname=False, split=False) + words = map(pyshlex.wordtree_as_string, wtrees) + assert len(words)==1 + return status, words[0] + + def expand_here_document(self, word): + """Return the expanded document as a single word. The here document is + assumed to be unquoted. + """ + status, wtrees = self._expand_word(word, pathname=False, + split=False, here_document=True) + words = map(pyshlex.wordtree_as_string, wtrees) + assert len(words)==1 + return words[0] + + def expand_redirection(self, word): + """Return a single word.""" + return self.expand_variable(word)[1] + + def get_env(self): + return self._env + + def _expand_word(self, token, pathname=True, split=True, here_document=False): + wtree = pyshlex.make_wordtree(token[1], here_document=here_document) + + # TODO: implement tilde expansion + def expand(wtree): + """Return a pseudo wordtree: the tree or its subelements can be empty + lists when no value result from the expansion. + """ + status = None + for part in wtree: + if not isinstance(part, list): + continue + if part[0]in ("'", '\\'): + continue + elif part[0] in ('`', '$('): + status, result = self._expand_command(part) + part[:] = result + elif part[0] in ('$', '${'): + part[:] = self._expand_parameter(part, wtree[0]=='"', split) + elif part[0] in ('', '"'): + status, result = expand(part) + part[:] = result + else: + raise NotImplementedError('%s expansion is not implemented' + % part[0]) + # [] is returned when an expansion result in no-field, + # like an empty $@ + wtree = [p for p in wtree if p != []] + if len(wtree) < 3: + return status, [] + return status, wtree + + status, wtree = expand(wtree) + if len(wtree) == 0: + return status, wtree + wtree = pyshlex.normalize_wordtree(wtree) + + if split: + wtrees = self._split_fields(wtree) + else: + wtrees = [wtree] + + if pathname: + wtrees = mappend(self._expand_pathname, wtrees) + + wtrees = map(self._remove_quotes, wtrees) + return status, wtrees + + def _expand_command(self, wtree): + # BUG: there is something to do with backslashes and quoted + # characters here + command = pyshlex.wordtree_as_string(wtree[1:-1]) + status, output = self.subshell_output(command) + return status, ['', output, ''] + + def _expand_parameter(self, wtree, quoted=False, split=False): + """Return a valid wtree or an empty list when no parameter results.""" + # Get the parameter name + # TODO: implement weird expansion rules with ':' + name = pyshlex.wordtree_as_string(wtree[1:-1]) + if not is_name(name) and not is_special_param(name): + raise ExpansionError('Bad substitution "%s"' % name) + # TODO: implement special parameters + if name in ('@', '*'): + args = self._env.get_positional_args() + if len(args) == 0: + return [] + if len(args)<2: + return ['', ''.join(args), ''] + + sep = self._env.get('IFS', '')[:1] + if split and quoted and name=='@': + # Introduce a new token to tell the caller that these parameters + # cause a split as specified in 2.5.2 + return ['@'] + args + [''] + else: + return ['', sep.join(args), ''] + + return ['', self._env.get(name, ''), ''] + + def _split_fields(self, wtree): + def is_empty(split): + return split==['', '', ''] + + def split_positional(quoted): + # Return a list of wtree split according positional parameters rules. + # All remaining '@' groups are removed. + assert quoted[0]=='"' + + splits = [[]] + for part in quoted: + if not isinstance(part, list) or part[0]!='@': + splits[-1].append(part) + else: + # Empty or single argument list were dealt with already + assert len(part)>3 + # First argument must join with the beginning part of the original word + splits[-1].append(part[1]) + # Create double-quotes expressions for every argument after the first + for arg in part[2:-1]: + splits[-1].append('"') + splits.append(['"', arg]) + return splits + + # At this point, all expansions but pathnames have occured. Only quoted + # and positional sequences remain. Thus, all candidates for field splitting + # are in the tree root, or are positional splits ('@') and lie in root + # children. + if not wtree or wtree[0] not in ('', '"'): + # The whole token is quoted or empty, nothing to split + return [wtree] + + if wtree[0]=='"': + wtree = ['', wtree, ''] + + result = [['', '']] + for part in wtree[1:-1]: + if isinstance(part, list): + if part[0]=='"': + splits = split_positional(part) + if len(splits)<=1: + result[-1] += [part, ''] + else: + # Terminate the current split + result[-1] += [splits[0], ''] + result += splits[1:-1] + # Create a new split + result += [['', splits[-1], '']] + else: + result[-1] += [part, ''] + else: + splits = self._env.split_fields(part) + if len(splits)<=1: + # No split + result[-1][-1] += part + else: + # Terminate the current resulting part and create a new one + result[-1][-1] += splits[0] + result[-1].append('') + result += [['', r, ''] for r in splits[1:-1]] + result += [['', splits[-1]]] + result[-1].append('') + + # Leading and trailing empty groups come from leading/trailing blanks + if result and is_empty(result[-1]): + result[-1:] = [] + if result and is_empty(result[0]): + result[:1] = [] + return result + + def _expand_pathname(self, wtree): + """See [2.6.6 Pathname Expansion].""" + if self._env.has_opt('-f'): + return [wtree] + + # All expansions have been performed, only quoted sequences should remain + # in the tree. Generate the pattern by folding the tree, escaping special + # characters when appear quoted + special_chars = '*?[]' + + def make_pattern(wtree): + subpattern = [] + for part in wtree[1:-1]: + if isinstance(part, list): + part = make_pattern(part) + elif wtree[0]!='': + for c in part: + # Meta-characters cannot be quoted + if c in special_chars: + raise GlobError() + subpattern.append(part) + return ''.join(subpattern) + + def pwd_glob(pattern): + cwd = os.getcwd() + os.chdir(self._env['PWD']) + try: + return glob.glob(pattern) + finally: + os.chdir(cwd) + + #TODO: check working directory issues here wrt relative patterns + try: + pattern = make_pattern(wtree) + paths = pwd_glob(pattern) + except GlobError: + # BUG: Meta-characters were found in quoted sequences. The should + # have been used literally but this is unsupported in current glob module. + # Instead we consider the whole tree must be used literally and + # therefore there is no point in globbing. This is wrong when meta + # characters are mixed with quoted meta in the same pattern like: + # < foo*"py*" > + paths = [] + + if not paths: + return [wtree] + return [['', path, ''] for path in paths] + + def _remove_quotes(self, wtree): + """See [2.6.7 Quote Removal].""" + + def unquote(wtree): + unquoted = [] + for part in wtree[1:-1]: + if isinstance(part, list): + part = unquote(part) + unquoted.append(part) + return ''.join(unquoted) + + return ['', unquote(wtree), ''] + + def subshell(self, script=None, ast=None, redirs=None): + """Execute the script or AST in a subshell, with inherited redirections + if redirs is not None. + """ + if redirs: + sub_redirs = redirs + else: + sub_redirs = redirs.clone() + + subshell = None + try: + subshell = Interpreter(None, self._debugflags, self._env.clone(True), + sub_redirs, opts=self._options) + return subshell.execute_script(script, ast) + finally: + if not redirs: sub_redirs.close() + if subshell: subshell.close() + + def subshell_output(self, script): + """Execute the script in a subshell and return the captured output.""" + # Create temporary file to capture subshell output + tmpfd, tmppath = tempfile.mkstemp() + try: + tmpfile = os.fdopen(tmpfd, 'wb') + stdout = FileWrapper('w', tmpfile) + + redirs = Redirections(self._redirs.stdin().dup(), + stdout, + self._redirs.stderr().dup()) + try: + status = self.subshell(script=script, redirs=redirs) + finally: + redirs.close() + redirs = None + + # Extract subshell standard output + tmpfile = open(tmppath, 'rb') + try: + output = tmpfile.read() + return status, output.rstrip('\n') + finally: + tmpfile.close() + finally: + os.remove(tmppath) + + def _asynclist(self, cmd): + args = (self._env.get_variables(), cmd) + arg = encodeargs(args) + assert len(args) < 30*1024 + cmd = ['pysh.bat', '--ast', '-c', arg] + p = subprocess.Popen(cmd, cwd=self._env['PWD']) + self._children[p.pid] = p + self._env['!'] = p.pid + return 0 + + def wait(self, pids=None): + if not pids: + pids = self._children.keys() + + status = 127 + for pid in pids: + if pid not in self._children: + continue + p = self._children.pop(pid) + status = p.wait() + + return status + diff --git a/bitbake/lib/pysh/lsprof.py b/bitbake/lib/pysh/lsprof.py new file mode 100644 index 0000000000..b1831c22a7 --- /dev/null +++ b/bitbake/lib/pysh/lsprof.py @@ -0,0 +1,116 @@ +#! /usr/bin/env python + +import sys +from _lsprof import Profiler, profiler_entry + +__all__ = ['profile', 'Stats'] + +def profile(f, *args, **kwds): + """XXX docstring""" + p = Profiler() + p.enable(subcalls=True, builtins=True) + try: + f(*args, **kwds) + finally: + p.disable() + return Stats(p.getstats()) + + +class Stats(object): + """XXX docstring""" + + def __init__(self, data): + self.data = data + + def sort(self, crit="inlinetime"): + """XXX docstring""" + if crit not in profiler_entry.__dict__: + raise ValueError("Can't sort by %s" % crit) + self.data.sort(lambda b, a: cmp(getattr(a, crit), + getattr(b, crit))) + for e in self.data: + if e.calls: + e.calls.sort(lambda b, a: cmp(getattr(a, crit), + getattr(b, crit))) + + def pprint(self, top=None, file=None, limit=None, climit=None): + """XXX docstring""" + if file is None: + file = sys.stdout + d = self.data + if top is not None: + d = d[:top] + cols = "% 12s %12s %11.4f %11.4f %s\n" + hcols = "% 12s %12s %12s %12s %s\n" + cols2 = "+%12s %12s %11.4f %11.4f + %s\n" + file.write(hcols % ("CallCount", "Recursive", "Total(ms)", + "Inline(ms)", "module:lineno(function)")) + count = 0 + for e in d: + file.write(cols % (e.callcount, e.reccallcount, e.totaltime, + e.inlinetime, label(e.code))) + count += 1 + if limit is not None and count == limit: + return + ccount = 0 + if e.calls: + for se in e.calls: + file.write(cols % ("+%s" % se.callcount, se.reccallcount, + se.totaltime, se.inlinetime, + "+%s" % label(se.code))) + count += 1 + ccount += 1 + if limit is not None and count == limit: + return + if climit is not None and ccount == climit: + break + + def freeze(self): + """Replace all references to code objects with string + descriptions; this makes it possible to pickle the instance.""" + + # this code is probably rather ickier than it needs to be! + for i in range(len(self.data)): + e = self.data[i] + if not isinstance(e.code, str): + self.data[i] = type(e)((label(e.code),) + e[1:]) + if e.calls: + for j in range(len(e.calls)): + se = e.calls[j] + if not isinstance(se.code, str): + e.calls[j] = type(se)((label(se.code),) + se[1:]) + +_fn2mod = {} + +def label(code): + if isinstance(code, str): + return code + try: + mname = _fn2mod[code.co_filename] + except KeyError: + for k, v in sys.modules.items(): + if v is None: + continue + if not hasattr(v, '__file__'): + continue + if not isinstance(v.__file__, str): + continue + if v.__file__.startswith(code.co_filename): + mname = _fn2mod[code.co_filename] = k + break + else: + mname = _fn2mod[code.co_filename] = '<%s>'%code.co_filename + + return '%s:%d(%s)' % (mname, code.co_firstlineno, code.co_name) + + +if __name__ == '__main__': + import os + sys.argv = sys.argv[1:] + if not sys.argv: + print >> sys.stderr, "usage: lsprof.py <script> <arguments...>" + sys.exit(2) + sys.path.insert(0, os.path.abspath(os.path.dirname(sys.argv[0]))) + stats = profile(execfile, sys.argv[0], globals(), locals()) + stats.sort() + stats.pprint() diff --git a/bitbake/lib/pysh/pysh.py b/bitbake/lib/pysh/pysh.py new file mode 100644 index 0000000000..b4e6145b51 --- /dev/null +++ b/bitbake/lib/pysh/pysh.py @@ -0,0 +1,167 @@ +# pysh.py - command processing for pysh. +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +import optparse +import os +import sys + +import interp + +SH_OPT = optparse.OptionParser(prog='pysh', usage="%prog [OPTIONS]", version='0.1') +SH_OPT.add_option('-c', action='store_true', dest='command_string', default=None, + help='A string that shall be interpreted by the shell as one or more commands') +SH_OPT.add_option('--redirect-to', dest='redirect_to', default=None, + help='Redirect script commands stdout and stderr to the specified file') +# See utility_command in builtin.py about the reason for this flag. +SH_OPT.add_option('--redirected', dest='redirected', action='store_true', default=False, + help='Tell the interpreter that stdout and stderr are actually the same objects, which is really stdout') +SH_OPT.add_option('--debug-parsing', action='store_true', dest='debug_parsing', default=False, + help='Trace PLY execution') +SH_OPT.add_option('--debug-tree', action='store_true', dest='debug_tree', default=False, + help='Display the generated syntax tree.') +SH_OPT.add_option('--debug-cmd', action='store_true', dest='debug_cmd', default=False, + help='Trace command execution before parameters expansion and exit status.') +SH_OPT.add_option('--debug-utility', action='store_true', dest='debug_utility', default=False, + help='Trace utility calls, after parameters expansions') +SH_OPT.add_option('--ast', action='store_true', dest='ast', default=False, + help='Encoded commands to execute in a subprocess') +SH_OPT.add_option('--profile', action='store_true', default=False, + help='Profile pysh run') + + +def split_args(args): + # Separate shell arguments from command ones + # Just stop at the first argument not starting with a dash. I know, this is completely broken, + # it ignores files starting with a dash or may take option values for command file. This is not + # supposed to happen for now + command_index = len(args) + for i,arg in enumerate(args): + if not arg.startswith('-'): + command_index = i + break + + return args[:command_index], args[command_index:] + + +def fixenv(env): + path = env.get('PATH') + if path is not None: + parts = path.split(os.pathsep) + # Remove Windows utilities from PATH, they are useless at best and + # some of them (find) may be confused with other utilities. + parts = [p for p in parts if 'system32' not in p.lower()] + env['PATH'] = os.pathsep.join(parts) + if env.get('HOME') is None: + # Several utilities, including cvsps, cannot work without + # a defined HOME directory. + env['HOME'] = os.path.expanduser('~') + return env + +def _sh(cwd, shargs, cmdargs, options, debugflags=None, env=None): + if os.environ.get('PYSH_TEXT') != '1': + import msvcrt + for fp in (sys.stdin, sys.stdout, sys.stderr): + msvcrt.setmode(fp.fileno(), os.O_BINARY) + + hgbin = os.environ.get('PYSH_HGTEXT') != '1' + + if debugflags is None: + debugflags = [] + if options.debug_parsing: debugflags.append('debug-parsing') + if options.debug_utility: debugflags.append('debug-utility') + if options.debug_cmd: debugflags.append('debug-cmd') + if options.debug_tree: debugflags.append('debug-tree') + + if env is None: + env = fixenv(dict(os.environ)) + if cwd is None: + cwd = os.getcwd() + + if not cmdargs: + # Nothing to do + return 0 + + ast = None + command_file = None + if options.command_string: + input = cmdargs[0] + if not options.ast: + input += '\n' + else: + args, input = interp.decodeargs(input), None + env, ast = args + cwd = env.get('PWD', cwd) + else: + command_file = cmdargs[0] + arguments = cmdargs[1:] + + prefix = interp.resolve_shebang(command_file, ignoreshell=True) + if prefix: + input = ' '.join(prefix + [command_file] + arguments) + else: + # Read commands from file + f = file(command_file) + try: + # Trailing newline to help the parser + input = f.read() + '\n' + finally: + f.close() + + redirect = None + try: + if options.redirected: + stdout = sys.stdout + stderr = stdout + elif options.redirect_to: + redirect = open(options.redirect_to, 'wb') + stdout = redirect + stderr = redirect + else: + stdout = sys.stdout + stderr = sys.stderr + + # TODO: set arguments to environment variables + opts = interp.Options() + opts.hgbinary = hgbin + ip = interp.Interpreter(cwd, debugflags, stdout=stdout, stderr=stderr, + opts=opts) + try: + # Export given environment in shell object + for k,v in env.iteritems(): + ip.get_env().export(k,v) + return ip.execute_script(input, ast, scriptpath=command_file) + finally: + ip.close() + finally: + if redirect is not None: + redirect.close() + +def sh(cwd=None, args=None, debugflags=None, env=None): + if args is None: + args = sys.argv[1:] + shargs, cmdargs = split_args(args) + options, shargs = SH_OPT.parse_args(shargs) + + if options.profile: + import lsprof + p = lsprof.Profiler() + p.enable(subcalls=True) + try: + return _sh(cwd, shargs, cmdargs, options, debugflags, env) + finally: + p.disable() + stats = lsprof.Stats(p.getstats()) + stats.sort() + stats.pprint(top=10, file=sys.stderr, climit=5) + else: + return _sh(cwd, shargs, cmdargs, options, debugflags, env) + +def main(): + sys.exit(sh()) + +if __name__=='__main__': + main() diff --git a/bitbake/lib/pysh/pyshlex.py b/bitbake/lib/pysh/pyshlex.py new file mode 100644 index 0000000000..b977b5e869 --- /dev/null +++ b/bitbake/lib/pysh/pyshlex.py @@ -0,0 +1,888 @@ +# pyshlex.py - PLY compatible lexer for pysh. +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +# TODO: +# - review all "char in 'abc'" snippets: the empty string can be matched +# - test line continuations within quoted/expansion strings +# - eof is buggy wrt sublexers +# - the lexer cannot really work in pull mode as it would be required to run +# PLY in pull mode. It was designed to work incrementally and it would not be +# that hard to enable pull mode. +import re +try: + s = set() + del s +except NameError: + from Set import Set as set + +from ply import lex +from sherrors import * + +class NeedMore(Exception): + pass + +def is_blank(c): + return c in (' ', '\t') + +_RE_DIGITS = re.compile(r'^\d+$') + +def are_digits(s): + return _RE_DIGITS.search(s) is not None + +_OPERATORS = dict([ + ('&&', 'AND_IF'), + ('||', 'OR_IF'), + (';;', 'DSEMI'), + ('<<', 'DLESS'), + ('>>', 'DGREAT'), + ('<&', 'LESSAND'), + ('>&', 'GREATAND'), + ('<>', 'LESSGREAT'), + ('<<-', 'DLESSDASH'), + ('>|', 'CLOBBER'), + ('&', 'AMP'), + (';', 'COMMA'), + ('<', 'LESS'), + ('>', 'GREATER'), + ('(', 'LPARENS'), + (')', 'RPARENS'), +]) + +#Make a function to silence pychecker "Local variable shadows global" +def make_partial_ops(): + partials = {} + for k in _OPERATORS: + for i in range(1, len(k)+1): + partials[k[:i]] = None + return partials + +_PARTIAL_OPERATORS = make_partial_ops() + +def is_partial_op(s): + """Return True if s matches a non-empty subpart of an operator starting + at its first character. + """ + return s in _PARTIAL_OPERATORS + +def is_op(s): + """If s matches an operator, returns the operator identifier. Return None + otherwise. + """ + return _OPERATORS.get(s) + +_RESERVEDS = dict([ + ('if', 'If'), + ('then', 'Then'), + ('else', 'Else'), + ('elif', 'Elif'), + ('fi', 'Fi'), + ('do', 'Do'), + ('done', 'Done'), + ('case', 'Case'), + ('esac', 'Esac'), + ('while', 'While'), + ('until', 'Until'), + ('for', 'For'), + ('{', 'Lbrace'), + ('}', 'Rbrace'), + ('!', 'Bang'), + ('in', 'In'), + ('|', 'PIPE'), +]) + +def get_reserved(s): + return _RESERVEDS.get(s) + +_RE_NAME = re.compile(r'^[0-9a-zA-Z_]+$') + +def is_name(s): + return _RE_NAME.search(s) is not None + +def find_chars(seq, chars): + for i,v in enumerate(seq): + if v in chars: + return i,v + return -1, None + +class WordLexer: + """WordLexer parse quoted or expansion expressions and return an expression + tree. The input string can be any well formed sequence beginning with quoting + or expansion character. Embedded expressions are handled recursively. The + resulting tree is made of lists and strings. Lists represent quoted or + expansion expressions. Each list first element is the opening separator, + the last one the closing separator. In-between can be any number of strings + or lists for sub-expressions. Non quoted/expansion expression can written as + strings or as lists with empty strings as starting and ending delimiters. + """ + + NAME_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_' + NAME_CHARSET = dict(zip(NAME_CHARSET, NAME_CHARSET)) + + SPECIAL_CHARSET = '@*#?-$!0' + + #Characters which can be escaped depends on the current delimiters + ESCAPABLE = { + '`': set(['$', '\\', '`']), + '"': set(['$', '\\', '`', '"']), + "'": set(), + } + + def __init__(self, heredoc = False): + # _buffer is the unprocessed input characters buffer + self._buffer = [] + # _stack is empty or contains a quoted list being processed + # (this is the DFS path to the quoted expression being evaluated). + self._stack = [] + self._escapable = None + # True when parsing unquoted here documents + self._heredoc = heredoc + + def add(self, data, eof=False): + """Feed the lexer with more data. If the quoted expression can be + delimited, return a tuple (expr, remaining) containing the expression + tree and the unconsumed data. + Otherwise, raise NeedMore. + """ + self._buffer += list(data) + self._parse(eof) + + result = self._stack[0] + remaining = ''.join(self._buffer) + self._stack = [] + self._buffer = [] + return result, remaining + + def _is_escapable(self, c, delim=None): + if delim is None: + if self._heredoc: + # Backslashes works as if they were double quoted in unquoted + # here-documents + delim = '"' + else: + if len(self._stack)<=1: + return True + delim = self._stack[-2][0] + + escapables = self.ESCAPABLE.get(delim, None) + return escapables is None or c in escapables + + def _parse_squote(self, buf, result, eof): + if not buf: + raise NeedMore() + try: + pos = buf.index("'") + except ValueError: + raise NeedMore() + result[-1] += ''.join(buf[:pos]) + result += ["'"] + return pos+1, True + + def _parse_bquote(self, buf, result, eof): + if not buf: + raise NeedMore() + + if buf[0]=='\n': + #Remove line continuations + result[:] = ['', '', ''] + elif self._is_escapable(buf[0]): + result[-1] += buf[0] + result += [''] + else: + #Keep as such + result[:] = ['', '\\'+buf[0], ''] + + return 1, True + + def _parse_dquote(self, buf, result, eof): + if not buf: + raise NeedMore() + pos, sep = find_chars(buf, '$\\`"') + if pos==-1: + raise NeedMore() + + result[-1] += ''.join(buf[:pos]) + if sep=='"': + result += ['"'] + return pos+1, True + else: + #Keep everything until the separator and defer processing + return pos, False + + def _parse_command(self, buf, result, eof): + if not buf: + raise NeedMore() + + chars = '$\\`"\'' + if result[0] == '$(': + chars += ')' + pos, sep = find_chars(buf, chars) + if pos == -1: + raise NeedMore() + + result[-1] += ''.join(buf[:pos]) + if (result[0]=='$(' and sep==')') or (result[0]=='`' and sep=='`'): + result += [sep] + return pos+1, True + else: + return pos, False + + def _parse_parameter(self, buf, result, eof): + if not buf: + raise NeedMore() + + pos, sep = find_chars(buf, '$\\`"\'}') + if pos==-1: + raise NeedMore() + + result[-1] += ''.join(buf[:pos]) + if sep=='}': + result += [sep] + return pos+1, True + else: + return pos, False + + def _parse_dollar(self, buf, result, eof): + sep = result[0] + if sep=='$': + if not buf: + #TODO: handle empty $ + raise NeedMore() + if buf[0]=='(': + if len(buf)==1: + raise NeedMore() + + if buf[1]=='(': + result[0] = '$((' + buf[:2] = [] + else: + result[0] = '$(' + buf[:1] = [] + + elif buf[0]=='{': + result[0] = '${' + buf[:1] = [] + else: + if buf[0] in self.SPECIAL_CHARSET: + result[-1] = buf[0] + read = 1 + else: + for read,c in enumerate(buf): + if c not in self.NAME_CHARSET: + break + else: + if not eof: + raise NeedMore() + read += 1 + + result[-1] += ''.join(buf[0:read]) + + if not result[-1]: + result[:] = ['', result[0], ''] + else: + result += [''] + return read,True + + sep = result[0] + if sep=='$(': + parsefunc = self._parse_command + elif sep=='${': + parsefunc = self._parse_parameter + else: + raise NotImplementedError() + + pos, closed = parsefunc(buf, result, eof) + return pos, closed + + def _parse(self, eof): + buf = self._buffer + stack = self._stack + recurse = False + + while 1: + if not stack or recurse: + if not buf: + raise NeedMore() + if buf[0] not in ('"\\`$\''): + raise ShellSyntaxError('Invalid quoted string sequence') + stack.append([buf[0], '']) + buf[:1] = [] + recurse = False + + result = stack[-1] + if result[0]=="'": + parsefunc = self._parse_squote + elif result[0]=='\\': + parsefunc = self._parse_bquote + elif result[0]=='"': + parsefunc = self._parse_dquote + elif result[0]=='`': + parsefunc = self._parse_command + elif result[0][0]=='$': + parsefunc = self._parse_dollar + else: + raise NotImplementedError() + + read, closed = parsefunc(buf, result, eof) + + buf[:read] = [] + if closed: + if len(stack)>1: + #Merge in parent expression + parsed = stack.pop() + stack[-1] += [parsed] + stack[-1] += [''] + else: + break + else: + recurse = True + +def normalize_wordtree(wtree): + """Fold back every literal sequence (delimited with empty strings) into + parent sequence. + """ + def normalize(wtree): + result = [] + for part in wtree[1:-1]: + if isinstance(part, list): + part = normalize(part) + if part[0]=='': + #Move the part content back at current level + result += part[1:-1] + continue + elif not part: + #Remove empty strings + continue + result.append(part) + if not result: + result = [''] + return [wtree[0]] + result + [wtree[-1]] + + return normalize(wtree) + + +def make_wordtree(token, here_document=False): + """Parse a delimited token and return a tree similar to the ones returned by + WordLexer. token may contain any combinations of expansion/quoted fields and + non-ones. + """ + tree = [''] + remaining = token + delimiters = '\\$`' + if not here_document: + delimiters += '\'"' + + while 1: + pos, sep = find_chars(remaining, delimiters) + if pos==-1: + tree += [remaining, ''] + return normalize_wordtree(tree) + tree.append(remaining[:pos]) + remaining = remaining[pos:] + + try: + result, remaining = WordLexer(heredoc = here_document).add(remaining, True) + except NeedMore: + raise ShellSyntaxError('Invalid token "%s"') + tree.append(result) + + +def wordtree_as_string(wtree): + """Rewrite an expression tree generated by make_wordtree as string.""" + def visit(node, output): + for child in node: + if isinstance(child, list): + visit(child, output) + else: + output.append(child) + + output = [] + visit(wtree, output) + return ''.join(output) + + +def unquote_wordtree(wtree): + """Fold the word tree while removing quotes everywhere. Other expansion + sequences are joined as such. + """ + def unquote(wtree): + unquoted = [] + if wtree[0] in ('', "'", '"', '\\'): + wtree = wtree[1:-1] + + for part in wtree: + if isinstance(part, list): + part = unquote(part) + unquoted.append(part) + return ''.join(unquoted) + + return unquote(wtree) + + +class HereDocLexer: + """HereDocLexer delimits whatever comes from the here-document starting newline + not included to the closing delimiter line included. + """ + def __init__(self, op, delim): + assert op in ('<<', '<<-') + if not delim: + raise ShellSyntaxError('invalid here document delimiter %s' % str(delim)) + + self._op = op + self._delim = delim + self._buffer = [] + self._token = [] + + def add(self, data, eof): + """If the here-document was delimited, return a tuple (content, remaining). + Raise NeedMore() otherwise. + """ + self._buffer += list(data) + self._parse(eof) + token = ''.join(self._token) + remaining = ''.join(self._buffer) + self._token, self._remaining = [], [] + return token, remaining + + def _parse(self, eof): + while 1: + #Look for first unescaped newline. Quotes may be ignored + escaped = False + for i,c in enumerate(self._buffer): + if escaped: + escaped = False + elif c=='\\': + escaped = True + elif c=='\n': + break + else: + i = -1 + + if i==-1 or self._buffer[i]!='\n': + if not eof: + raise NeedMore() + #No more data, maybe the last line is closing delimiter + line = ''.join(self._buffer) + eol = '' + self._buffer[:] = [] + else: + line = ''.join(self._buffer[:i]) + eol = self._buffer[i] + self._buffer[:i+1] = [] + + if self._op=='<<-': + line = line.lstrip('\t') + + if line==self._delim: + break + + self._token += [line, eol] + if i==-1: + break + +class Token: + #TODO: check this is still in use + OPERATOR = 'OPERATOR' + WORD = 'WORD' + + def __init__(self): + self.value = '' + self.type = None + + def __getitem__(self, key): + #Behave like a two elements tuple + if key==0: + return self.type + if key==1: + return self.value + raise IndexError(key) + + +class HereDoc: + def __init__(self, op, name=None): + self.op = op + self.name = name + self.pendings = [] + +TK_COMMA = 'COMMA' +TK_AMPERSAND = 'AMP' +TK_OP = 'OP' +TK_TOKEN = 'TOKEN' +TK_COMMENT = 'COMMENT' +TK_NEWLINE = 'NEWLINE' +TK_IONUMBER = 'IO_NUMBER' +TK_ASSIGNMENT = 'ASSIGNMENT_WORD' +TK_HERENAME = 'HERENAME' + +class Lexer: + """Main lexer. + + Call add() until the script AST is returned. + """ + # Here-document handling makes the whole thing more complex because they basically + # force tokens to be reordered: here-content must come right after the operator + # and the here-document name, while some other tokens might be following the + # here-document expression on the same line. + # + # So, here-doc states are basically: + # *self._state==ST_NORMAL + # - self._heredoc.op is None: no here-document + # - self._heredoc.op is not None but name is: here-document operator matched, + # waiting for the document name/delimiter + # - self._heredoc.op and name are not None: here-document is ready, following + # tokens are being stored and will be pushed again when the document is + # completely parsed. + # *self._state==ST_HEREDOC + # - The here-document is being delimited by self._herelexer. Once it is done + # the content is pushed in front of the pending token list then all these + # tokens are pushed once again. + ST_NORMAL = 'ST_NORMAL' + ST_OP = 'ST_OP' + ST_BACKSLASH = 'ST_BACKSLASH' + ST_QUOTED = 'ST_QUOTED' + ST_COMMENT = 'ST_COMMENT' + ST_HEREDOC = 'ST_HEREDOC' + + #Match end of backquote strings + RE_BACKQUOTE_END = re.compile(r'(?<!\\)(`)') + + def __init__(self, parent_state = None): + self._input = [] + self._pos = 0 + + self._token = '' + self._type = TK_TOKEN + + self._state = self.ST_NORMAL + self._parent_state = parent_state + self._wordlexer = None + + self._heredoc = HereDoc(None) + self._herelexer = None + + ### Following attributes are not used for delimiting token and can safely + ### be changed after here-document detection (see _push_toke) + + # Count the number of tokens following a 'For' reserved word. Needed to + # return an 'In' reserved word if it comes in third place. + self._for_count = None + + def add(self, data, eof=False): + """Feed the lexer with data. + + When eof is set to True, returns unconsumed data or raise if the lexer + is in the middle of a delimiting operation. + Raise NeedMore otherwise. + """ + self._input += list(data) + self._parse(eof) + self._input[:self._pos] = [] + return ''.join(self._input) + + def _parse(self, eof): + while self._state: + if self._pos>=len(self._input): + if not eof: + raise NeedMore() + elif self._state not in (self.ST_OP, self.ST_QUOTED, self.ST_HEREDOC): + #Delimit the current token and leave cleanly + self._push_token('') + break + else: + #Let the sublexer handle the eof themselves + pass + + if self._state==self.ST_NORMAL: + self._parse_normal() + elif self._state==self.ST_COMMENT: + self._parse_comment() + elif self._state==self.ST_OP: + self._parse_op(eof) + elif self._state==self.ST_QUOTED: + self._parse_quoted(eof) + elif self._state==self.ST_HEREDOC: + self._parse_heredoc(eof) + else: + assert False, "Unknown state " + str(self._state) + + if self._heredoc.op is not None: + raise ShellSyntaxError('missing here-document delimiter') + + def _parse_normal(self): + c = self._input[self._pos] + if c=='\n': + self._push_token(c) + self._token = c + self._type = TK_NEWLINE + self._push_token('') + self._pos += 1 + elif c in ('\\', '\'', '"', '`', '$'): + self._state = self.ST_QUOTED + elif is_partial_op(c): + self._push_token(c) + + self._type = TK_OP + self._token += c + self._pos += 1 + self._state = self.ST_OP + elif is_blank(c): + self._push_token(c) + + #Discard blanks + self._pos += 1 + elif self._token: + self._token += c + self._pos += 1 + elif c=='#': + self._state = self.ST_COMMENT + self._type = TK_COMMENT + self._pos += 1 + else: + self._pos += 1 + self._token += c + + def _parse_op(self, eof): + assert self._token + + while 1: + if self._pos>=len(self._input): + if not eof: + raise NeedMore() + c = '' + else: + c = self._input[self._pos] + + op = self._token + c + if c and is_partial_op(op): + #Still parsing an operator + self._token = op + self._pos += 1 + else: + #End of operator + self._push_token(c) + self._state = self.ST_NORMAL + break + + def _parse_comment(self): + while 1: + if self._pos>=len(self._input): + raise NeedMore() + + c = self._input[self._pos] + if c=='\n': + #End of comment, do not consume the end of line + self._state = self.ST_NORMAL + break + else: + self._token += c + self._pos += 1 + + def _parse_quoted(self, eof): + """Precondition: the starting backquote/dollar is still in the input queue.""" + if not self._wordlexer: + self._wordlexer = WordLexer() + + if self._pos<len(self._input): + #Transfer input queue character into the subparser + input = self._input[self._pos:] + self._pos += len(input) + + wtree, remaining = self._wordlexer.add(input, eof) + self._wordlexer = None + self._token += wordtree_as_string(wtree) + + #Put unparsed character back in the input queue + if remaining: + self._input[self._pos:self._pos] = list(remaining) + self._state = self.ST_NORMAL + + def _parse_heredoc(self, eof): + assert not self._token + + if self._herelexer is None: + self._herelexer = HereDocLexer(self._heredoc.op, self._heredoc.name) + + if self._pos<len(self._input): + #Transfer input queue character into the subparser + input = self._input[self._pos:] + self._pos += len(input) + + self._token, remaining = self._herelexer.add(input, eof) + + #Reset here-document state + self._herelexer = None + heredoc, self._heredoc = self._heredoc, HereDoc(None) + if remaining: + self._input[self._pos:self._pos] = list(remaining) + self._state = self.ST_NORMAL + + #Push pending tokens + heredoc.pendings[:0] = [(self._token, self._type, heredoc.name)] + for token, type, delim in heredoc.pendings: + self._token = token + self._type = type + self._push_token(delim) + + def _push_token(self, delim): + if not self._token: + return 0 + + if self._heredoc.op is not None: + if self._heredoc.name is None: + #Here-document name + if self._type!=TK_TOKEN: + raise ShellSyntaxError("expecting here-document name, got '%s'" % self._token) + self._heredoc.name = unquote_wordtree(make_wordtree(self._token)) + self._type = TK_HERENAME + else: + #Capture all tokens until the newline starting the here-document + if self._type==TK_NEWLINE: + assert self._state==self.ST_NORMAL + self._state = self.ST_HEREDOC + + self._heredoc.pendings.append((self._token, self._type, delim)) + self._token = '' + self._type = TK_TOKEN + return 1 + + # BEWARE: do not change parser state from here to the end of the function: + # when parsing between an here-document operator to the end of the line + # tokens are stored in self._heredoc.pendings. Therefore, they will not + # reach the section below. + + #Check operators + if self._type==TK_OP: + #False positive because of partial op matching + op = is_op(self._token) + if not op: + self._type = TK_TOKEN + else: + #Map to the specific operator + self._type = op + if self._token in ('<<', '<<-'): + #Done here rather than in _parse_op because there is no need + #to change the parser state since we are still waiting for + #the here-document name + if self._heredoc.op is not None: + raise ShellSyntaxError("syntax error near token '%s'" % self._token) + assert self._heredoc.op is None + self._heredoc.op = self._token + + if self._type==TK_TOKEN: + if '=' in self._token and not delim: + if self._token.startswith('='): + #Token is a WORD... a TOKEN that is. + pass + else: + prev = self._token[:self._token.find('=')] + if is_name(prev): + self._type = TK_ASSIGNMENT + else: + #Just a token (unspecified) + pass + else: + reserved = get_reserved(self._token) + if reserved is not None: + if reserved=='In' and self._for_count!=2: + #Sorry, not a reserved word after all + pass + else: + self._type = reserved + if reserved in ('For', 'Case'): + self._for_count = 0 + elif are_digits(self._token) and delim in ('<', '>'): + #Detect IO_NUMBER + self._type = TK_IONUMBER + elif self._token==';': + self._type = TK_COMMA + elif self._token=='&': + self._type = TK_AMPERSAND + elif self._type==TK_COMMENT: + #Comments are not part of sh grammar, ignore them + self._token = '' + self._type = TK_TOKEN + return 0 + + if self._for_count is not None: + #Track token count in 'For' expression to detect 'In' reserved words. + #Can only be in third position, no need to go beyond + self._for_count += 1 + if self._for_count==3: + self._for_count = None + + self.on_token((self._token, self._type)) + self._token = '' + self._type = TK_TOKEN + return 1 + + def on_token(self, token): + raise NotImplementedError + + +tokens = [ + TK_TOKEN, +# To silence yacc unused token warnings +# TK_COMMENT, + TK_NEWLINE, + TK_IONUMBER, + TK_ASSIGNMENT, + TK_HERENAME, +] + +#Add specific operators +tokens += _OPERATORS.values() +#Add reserved words +tokens += _RESERVEDS.values() + +class PLYLexer(Lexer): + """Bridge Lexer and PLY lexer interface.""" + def __init__(self): + Lexer.__init__(self) + self._tokens = [] + self._current = 0 + self.lineno = 0 + + def on_token(self, token): + value, type = token + + self.lineno = 0 + t = lex.LexToken() + t.value = value + t.type = type + t.lexer = self + t.lexpos = 0 + t.lineno = 0 + + self._tokens.append(t) + + def is_empty(self): + return not bool(self._tokens) + + #PLY compliant interface + def token(self): + if self._current>=len(self._tokens): + return None + t = self._tokens[self._current] + self._current += 1 + return t + + +def get_tokens(s): + """Parse the input string and return a tuple (tokens, unprocessed) where + tokens is a list of parsed tokens and unprocessed is the part of the input + string left untouched by the lexer. + """ + lexer = PLYLexer() + untouched = lexer.add(s, True) + tokens = [] + while 1: + token = lexer.token() + if token is None: + break + tokens.append(token) + + tokens = [(t.value, t.type) for t in tokens] + return tokens, untouched diff --git a/bitbake/lib/pysh/pyshyacc.py b/bitbake/lib/pysh/pyshyacc.py new file mode 100644 index 0000000000..3d9510c0c3 --- /dev/null +++ b/bitbake/lib/pysh/pyshyacc.py @@ -0,0 +1,772 @@ +# pyshyacc.py - PLY grammar definition for pysh +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +"""PLY grammar file. +""" +import sys + +import pyshlex +tokens = pyshlex.tokens + +from ply import yacc +import sherrors + +class IORedirect: + def __init__(self, op, filename, io_number=None): + self.op = op + self.filename = filename + self.io_number = io_number + +class HereDocument: + def __init__(self, op, name, content, io_number=None): + self.op = op + self.name = name + self.content = content + self.io_number = io_number + +def make_io_redirect(p): + """Make an IORedirect instance from the input 'io_redirect' production.""" + name, io_number, io_target = p + assert name=='io_redirect' + + if io_target[0]=='io_file': + io_type, io_op, io_file = io_target + return IORedirect(io_op, io_file, io_number) + elif io_target[0]=='io_here': + io_type, io_op, io_name, io_content = io_target + return HereDocument(io_op, io_name, io_content, io_number) + else: + assert False, "Invalid IO redirection token %s" % repr(io_type) + +class SimpleCommand: + """ + assigns contains (name, value) pairs. + """ + def __init__(self, words, redirs, assigns): + self.words = list(words) + self.redirs = list(redirs) + self.assigns = list(assigns) + +class Pipeline: + def __init__(self, commands, reverse_status=False): + self.commands = list(commands) + assert self.commands #Grammar forbids this + self.reverse_status = reverse_status + +class AndOr: + def __init__(self, op, left, right): + self.op = str(op) + self.left = left + self.right = right + +class ForLoop: + def __init__(self, name, items, cmds): + self.name = str(name) + self.items = list(items) + self.cmds = list(cmds) + +class WhileLoop: + def __init__(self, condition, cmds): + self.condition = list(condition) + self.cmds = list(cmds) + +class UntilLoop: + def __init__(self, condition, cmds): + self.condition = list(condition) + self.cmds = list(cmds) + +class FunDef: + def __init__(self, name, body): + self.name = str(name) + self.body = body + +class BraceGroup: + def __init__(self, cmds): + self.cmds = list(cmds) + +class IfCond: + def __init__(self, cond, if_cmds, else_cmds): + self.cond = list(cond) + self.if_cmds = if_cmds + self.else_cmds = else_cmds + +class Case: + def __init__(self, name, items): + self.name = name + self.items = items + +class SubShell: + def __init__(self, cmds): + self.cmds = cmds + +class RedirectList: + def __init__(self, cmd, redirs): + self.cmd = cmd + self.redirs = list(redirs) + +def get_production(productions, ptype): + """productions must be a list of production tuples like (name, obj) where + name is the production string identifier. + Return the first production named 'ptype'. Raise KeyError if None can be + found. + """ + for production in productions: + if production is not None and production[0]==ptype: + return production + raise KeyError(ptype) + +#------------------------------------------------------------------------------- +# PLY grammar definition +#------------------------------------------------------------------------------- + +def p_multiple_commands(p): + """multiple_commands : newline_sequence + | complete_command + | multiple_commands complete_command""" + if len(p)==2: + if p[1] is not None: + p[0] = [p[1]] + else: + p[0] = [] + else: + p[0] = p[1] + [p[2]] + +def p_complete_command(p): + """complete_command : list separator + | list""" + if len(p)==3 and p[2] and p[2][1] == '&': + p[0] = ('async', p[1]) + else: + p[0] = p[1] + +def p_list(p): + """list : list separator_op and_or + | and_or""" + if len(p)==2: + p[0] = [p[1]] + else: + #if p[2]!=';': + # raise NotImplementedError('AND-OR list asynchronous execution is not implemented') + p[0] = p[1] + [p[3]] + +def p_and_or(p): + """and_or : pipeline + | and_or AND_IF linebreak pipeline + | and_or OR_IF linebreak pipeline""" + if len(p)==2: + p[0] = p[1] + else: + p[0] = ('and_or', AndOr(p[2], p[1], p[4])) + +def p_maybe_bang_word(p): + """maybe_bang_word : Bang""" + p[0] = ('maybe_bang_word', p[1]) + +def p_pipeline(p): + """pipeline : pipe_sequence + | bang_word pipe_sequence""" + if len(p)==3: + p[0] = ('pipeline', Pipeline(p[2][1:], True)) + else: + p[0] = ('pipeline', Pipeline(p[1][1:])) + +def p_pipe_sequence(p): + """pipe_sequence : command + | pipe_sequence PIPE linebreak command""" + if len(p)==2: + p[0] = ['pipe_sequence', p[1]] + else: + p[0] = p[1] + [p[4]] + +def p_command(p): + """command : simple_command + | compound_command + | compound_command redirect_list + | function_definition""" + + if p[1][0] in ( 'simple_command', + 'for_clause', + 'while_clause', + 'until_clause', + 'case_clause', + 'if_clause', + 'function_definition', + 'subshell', + 'brace_group',): + if len(p) == 2: + p[0] = p[1] + else: + p[0] = ('redirect_list', RedirectList(p[1], p[2][1:])) + else: + raise NotImplementedError('%s command is not implemented' % repr(p[1][0])) + +def p_compound_command(p): + """compound_command : brace_group + | subshell + | for_clause + | case_clause + | if_clause + | while_clause + | until_clause""" + p[0] = p[1] + +def p_subshell(p): + """subshell : LPARENS compound_list RPARENS""" + p[0] = ('subshell', SubShell(p[2][1:])) + +def p_compound_list(p): + """compound_list : term + | newline_list term + | term separator + | newline_list term separator""" + productions = p[1:] + try: + sep = get_production(productions, 'separator') + if sep[1]!=';': + raise NotImplementedError() + except KeyError: + pass + term = get_production(productions, 'term') + p[0] = ['compound_list'] + term[1:] + +def p_term(p): + """term : term separator and_or + | and_or""" + if len(p)==2: + p[0] = ['term', p[1]] + else: + if p[2] is not None and p[2][1] == '&': + p[0] = ['term', ('async', p[1][1:])] + [p[3]] + else: + p[0] = p[1] + [p[3]] + +def p_maybe_for_word(p): + # Rearrange 'For' priority wrt TOKEN. See p_for_word + """maybe_for_word : For""" + p[0] = ('maybe_for_word', p[1]) + +def p_for_clause(p): + """for_clause : for_word name linebreak do_group + | for_word name linebreak in sequential_sep do_group + | for_word name linebreak in wordlist sequential_sep do_group""" + productions = p[1:] + do_group = get_production(productions, 'do_group') + try: + items = get_production(productions, 'in')[1:] + except KeyError: + raise NotImplementedError('"in" omission is not implemented') + + try: + items = get_production(productions, 'wordlist')[1:] + except KeyError: + items = [] + + name = p[2] + p[0] = ('for_clause', ForLoop(name, items, do_group[1:])) + +def p_name(p): + """name : token""" #Was NAME instead of token + p[0] = p[1] + +def p_in(p): + """in : In""" + p[0] = ('in', p[1]) + +def p_wordlist(p): + """wordlist : wordlist token + | token""" + if len(p)==2: + p[0] = ['wordlist', ('TOKEN', p[1])] + else: + p[0] = p[1] + [('TOKEN', p[2])] + +def p_case_clause(p): + """case_clause : Case token linebreak in linebreak case_list Esac + | Case token linebreak in linebreak case_list_ns Esac + | Case token linebreak in linebreak Esac""" + if len(p) < 8: + items = [] + else: + items = p[6][1:] + name = p[2] + p[0] = ('case_clause', Case(name, [c[1] for c in items])) + +def p_case_list_ns(p): + """case_list_ns : case_list case_item_ns + | case_item_ns""" + p_case_list(p) + +def p_case_list(p): + """case_list : case_list case_item + | case_item""" + if len(p)==2: + p[0] = ['case_list', p[1]] + else: + p[0] = p[1] + [p[2]] + +def p_case_item_ns(p): + """case_item_ns : pattern RPARENS linebreak + | pattern RPARENS compound_list linebreak + | LPARENS pattern RPARENS linebreak + | LPARENS pattern RPARENS compound_list linebreak""" + p_case_item(p) + +def p_case_item(p): + """case_item : pattern RPARENS linebreak DSEMI linebreak + | pattern RPARENS compound_list DSEMI linebreak + | LPARENS pattern RPARENS linebreak DSEMI linebreak + | LPARENS pattern RPARENS compound_list DSEMI linebreak""" + if len(p) < 7: + name = p[1][1:] + else: + name = p[2][1:] + + try: + cmds = get_production(p[1:], "compound_list")[1:] + except KeyError: + cmds = [] + + p[0] = ('case_item', (name, cmds)) + +def p_pattern(p): + """pattern : token + | pattern PIPE token""" + if len(p)==2: + p[0] = ['pattern', ('TOKEN', p[1])] + else: + p[0] = p[1] + [('TOKEN', p[2])] + +def p_maybe_if_word(p): + # Rearrange 'If' priority wrt TOKEN. See p_if_word + """maybe_if_word : If""" + p[0] = ('maybe_if_word', p[1]) + +def p_maybe_then_word(p): + # Rearrange 'Then' priority wrt TOKEN. See p_then_word + """maybe_then_word : Then""" + p[0] = ('maybe_then_word', p[1]) + +def p_if_clause(p): + """if_clause : if_word compound_list then_word compound_list else_part Fi + | if_word compound_list then_word compound_list Fi""" + else_part = [] + if len(p)==7: + else_part = p[5] + p[0] = ('if_clause', IfCond(p[2][1:], p[4][1:], else_part)) + +def p_else_part(p): + """else_part : Elif compound_list then_word compound_list else_part + | Elif compound_list then_word compound_list + | Else compound_list""" + if len(p)==3: + p[0] = p[2][1:] + else: + else_part = [] + if len(p)==6: + else_part = p[5] + p[0] = ('elif', IfCond(p[2][1:], p[4][1:], else_part)) + +def p_while_clause(p): + """while_clause : While compound_list do_group""" + p[0] = ('while_clause', WhileLoop(p[2][1:], p[3][1:])) + +def p_maybe_until_word(p): + # Rearrange 'Until' priority wrt TOKEN. See p_until_word + """maybe_until_word : Until""" + p[0] = ('maybe_until_word', p[1]) + +def p_until_clause(p): + """until_clause : until_word compound_list do_group""" + p[0] = ('until_clause', UntilLoop(p[2][1:], p[3][1:])) + +def p_function_definition(p): + """function_definition : fname LPARENS RPARENS linebreak function_body""" + p[0] = ('function_definition', FunDef(p[1], p[5])) + +def p_function_body(p): + """function_body : compound_command + | compound_command redirect_list""" + if len(p)!=2: + raise NotImplementedError('functions redirections lists are not implemented') + p[0] = p[1] + +def p_fname(p): + """fname : TOKEN""" #Was NAME instead of token + p[0] = p[1] + +def p_brace_group(p): + """brace_group : Lbrace compound_list Rbrace""" + p[0] = ('brace_group', BraceGroup(p[2][1:])) + +def p_maybe_done_word(p): + #See p_assignment_word for details. + """maybe_done_word : Done""" + p[0] = ('maybe_done_word', p[1]) + +def p_maybe_do_word(p): + """maybe_do_word : Do""" + p[0] = ('maybe_do_word', p[1]) + +def p_do_group(p): + """do_group : do_word compound_list done_word""" + #Do group contains a list of AndOr + p[0] = ['do_group'] + p[2][1:] + +def p_simple_command(p): + """simple_command : cmd_prefix cmd_word cmd_suffix + | cmd_prefix cmd_word + | cmd_prefix + | cmd_name cmd_suffix + | cmd_name""" + words, redirs, assigns = [], [], [] + for e in p[1:]: + name = e[0] + if name in ('cmd_prefix', 'cmd_suffix'): + for sube in e[1:]: + subname = sube[0] + if subname=='io_redirect': + redirs.append(make_io_redirect(sube)) + elif subname=='ASSIGNMENT_WORD': + assigns.append(sube) + else: + words.append(sube) + elif name in ('cmd_word', 'cmd_name'): + words.append(e) + + cmd = SimpleCommand(words, redirs, assigns) + p[0] = ('simple_command', cmd) + +def p_cmd_name(p): + """cmd_name : TOKEN""" + p[0] = ('cmd_name', p[1]) + +def p_cmd_word(p): + """cmd_word : token""" + p[0] = ('cmd_word', p[1]) + +def p_maybe_assignment_word(p): + #See p_assignment_word for details. + """maybe_assignment_word : ASSIGNMENT_WORD""" + p[0] = ('maybe_assignment_word', p[1]) + +def p_cmd_prefix(p): + """cmd_prefix : io_redirect + | cmd_prefix io_redirect + | assignment_word + | cmd_prefix assignment_word""" + try: + prefix = get_production(p[1:], 'cmd_prefix') + except KeyError: + prefix = ['cmd_prefix'] + + try: + value = get_production(p[1:], 'assignment_word')[1] + value = ('ASSIGNMENT_WORD', value.split('=', 1)) + except KeyError: + value = get_production(p[1:], 'io_redirect') + p[0] = prefix + [value] + +def p_cmd_suffix(p): + """cmd_suffix : io_redirect + | cmd_suffix io_redirect + | token + | cmd_suffix token + | maybe_for_word + | cmd_suffix maybe_for_word + | maybe_done_word + | cmd_suffix maybe_done_word + | maybe_do_word + | cmd_suffix maybe_do_word + | maybe_until_word + | cmd_suffix maybe_until_word + | maybe_assignment_word + | cmd_suffix maybe_assignment_word + | maybe_if_word + | cmd_suffix maybe_if_word + | maybe_then_word + | cmd_suffix maybe_then_word + | maybe_bang_word + | cmd_suffix maybe_bang_word""" + try: + suffix = get_production(p[1:], 'cmd_suffix') + token = p[2] + except KeyError: + suffix = ['cmd_suffix'] + token = p[1] + + if isinstance(token, tuple): + if token[0]=='io_redirect': + p[0] = suffix + [token] + else: + #Convert maybe_* to TOKEN if necessary + p[0] = suffix + [('TOKEN', token[1])] + else: + p[0] = suffix + [('TOKEN', token)] + +def p_redirect_list(p): + """redirect_list : io_redirect + | redirect_list io_redirect""" + if len(p) == 2: + p[0] = ['redirect_list', make_io_redirect(p[1])] + else: + p[0] = p[1] + [make_io_redirect(p[2])] + +def p_io_redirect(p): + """io_redirect : io_file + | IO_NUMBER io_file + | io_here + | IO_NUMBER io_here""" + if len(p)==3: + p[0] = ('io_redirect', p[1], p[2]) + else: + p[0] = ('io_redirect', None, p[1]) + +def p_io_file(p): + #Return the tuple (operator, filename) + """io_file : LESS filename + | LESSAND filename + | GREATER filename + | GREATAND filename + | DGREAT filename + | LESSGREAT filename + | CLOBBER filename""" + #Extract the filename from the file + p[0] = ('io_file', p[1], p[2][1]) + +def p_filename(p): + #Return the filename + """filename : TOKEN""" + p[0] = ('filename', p[1]) + +def p_io_here(p): + """io_here : DLESS here_end + | DLESSDASH here_end""" + p[0] = ('io_here', p[1], p[2][1], p[2][2]) + +def p_here_end(p): + """here_end : HERENAME TOKEN""" + p[0] = ('here_document', p[1], p[2]) + +def p_newline_sequence(p): + # Nothing in the grammar can handle leading NEWLINE productions, so add + # this one with the lowest possible priority relatively to newline_list. + """newline_sequence : newline_list""" + p[0] = None + +def p_newline_list(p): + """newline_list : NEWLINE + | newline_list NEWLINE""" + p[0] = None + +def p_linebreak(p): + """linebreak : newline_list + | empty""" + p[0] = None + +def p_separator_op(p): + """separator_op : COMMA + | AMP""" + p[0] = p[1] + +def p_separator(p): + """separator : separator_op linebreak + | newline_list""" + if len(p)==2: + #Ignore newlines + p[0] = None + else: + #Keep the separator operator + p[0] = ('separator', p[1]) + +def p_sequential_sep(p): + """sequential_sep : COMMA linebreak + | newline_list""" + p[0] = None + +# Low priority TOKEN => for_word conversion. +# Let maybe_for_word be used as a token when necessary in higher priority +# rules. +def p_for_word(p): + """for_word : maybe_for_word""" + p[0] = p[1] + +def p_if_word(p): + """if_word : maybe_if_word""" + p[0] = p[1] + +def p_then_word(p): + """then_word : maybe_then_word""" + p[0] = p[1] + +def p_done_word(p): + """done_word : maybe_done_word""" + p[0] = p[1] + +def p_do_word(p): + """do_word : maybe_do_word""" + p[0] = p[1] + +def p_until_word(p): + """until_word : maybe_until_word""" + p[0] = p[1] + +def p_assignment_word(p): + """assignment_word : maybe_assignment_word""" + p[0] = ('assignment_word', p[1][1]) + +def p_bang_word(p): + """bang_word : maybe_bang_word""" + p[0] = ('bang_word', p[1][1]) + +def p_token(p): + """token : TOKEN + | Fi""" + p[0] = p[1] + +def p_empty(p): + 'empty :' + p[0] = None + +# Error rule for syntax errors +def p_error(p): + msg = [] + w = msg.append + w('%r\n' % p) + w('followed by:\n') + for i in range(5): + n = yacc.token() + if not n: + break + w(' %r\n' % n) + raise sherrors.ShellSyntaxError(''.join(msg)) + +# Build the parser +try: + import pyshtables +except ImportError: + yacc.yacc(tabmodule = 'pyshtables') +else: + yacc.yacc(tabmodule = 'pysh.pyshtables', write_tables = 0, debug = 0) + + +def parse(input, eof=False, debug=False): + """Parse a whole script at once and return the generated AST and unconsumed + data in a tuple. + + NOTE: eof is probably meaningless for now, the parser being unable to work + in pull mode. It should be set to True. + """ + lexer = pyshlex.PLYLexer() + remaining = lexer.add(input, eof) + if lexer.is_empty(): + return [], remaining + if debug: + debug = 2 + return yacc.parse(lexer=lexer, debug=debug), remaining + +#------------------------------------------------------------------------------- +# AST rendering helpers +#------------------------------------------------------------------------------- + +def format_commands(v): + """Return a tree made of strings and lists. Make command trees easier to + display. + """ + if isinstance(v, list): + return [format_commands(c) for c in v] + if isinstance(v, tuple): + if len(v)==2 and isinstance(v[0], str) and not isinstance(v[1], str): + if v[0] == 'async': + return ['AsyncList', map(format_commands, v[1])] + else: + #Avoid decomposing tuples like ('pipeline', Pipeline(...)) + return format_commands(v[1]) + return format_commands(list(v)) + elif isinstance(v, IfCond): + name = ['IfCond'] + name += ['if', map(format_commands, v.cond)] + name += ['then', map(format_commands, v.if_cmds)] + name += ['else', map(format_commands, v.else_cmds)] + return name + elif isinstance(v, ForLoop): + name = ['ForLoop'] + name += [repr(v.name)+' in ', map(str, v.items)] + name += ['commands', map(format_commands, v.cmds)] + return name + elif isinstance(v, AndOr): + return [v.op, format_commands(v.left), format_commands(v.right)] + elif isinstance(v, Pipeline): + name = 'Pipeline' + if v.reverse_status: + name = '!' + name + return [name, format_commands(v.commands)] + elif isinstance(v, SimpleCommand): + name = ['SimpleCommand'] + if v.words: + name += ['words', map(str, v.words)] + if v.assigns: + assigns = [tuple(a[1]) for a in v.assigns] + name += ['assigns', map(str, assigns)] + if v.redirs: + name += ['redirs', map(format_commands, v.redirs)] + return name + elif isinstance(v, RedirectList): + name = ['RedirectList'] + if v.redirs: + name += ['redirs', map(format_commands, v.redirs)] + name += ['command', format_commands(v.cmd)] + return name + elif isinstance(v, IORedirect): + return ' '.join(map(str, (v.io_number, v.op, v.filename))) + elif isinstance(v, HereDocument): + return ' '.join(map(str, (v.io_number, v.op, repr(v.name), repr(v.content)))) + elif isinstance(v, SubShell): + return ['SubShell', map(format_commands, v.cmds)] + else: + return repr(v) + +def print_commands(cmds, output=sys.stdout): + """Pretty print a command tree.""" + def print_tree(cmd, spaces, output): + if isinstance(cmd, list): + for c in cmd: + print_tree(c, spaces + 3, output) + else: + print >>output, ' '*spaces + str(cmd) + + formatted = format_commands(cmds) + print_tree(formatted, 0, output) + + +def stringify_commands(cmds): + """Serialize a command tree as a string. + + Returned string is not pretty and is currently used for unit tests only. + """ + def stringify(value): + output = [] + if isinstance(value, list): + formatted = [] + for v in value: + formatted.append(stringify(v)) + formatted = ' '.join(formatted) + output.append(''.join(['<', formatted, '>'])) + else: + output.append(value) + return ' '.join(output) + + return stringify(format_commands(cmds)) + + +def visit_commands(cmds, callable): + """Visit the command tree and execute callable on every Pipeline and + SimpleCommand instances. + """ + if isinstance(cmds, (tuple, list)): + map(lambda c: visit_commands(c,callable), cmds) + elif isinstance(cmds, (Pipeline, SimpleCommand)): + callable(cmds) diff --git a/bitbake/lib/pysh/sherrors.py b/bitbake/lib/pysh/sherrors.py new file mode 100644 index 0000000000..1d5bd53b3a --- /dev/null +++ b/bitbake/lib/pysh/sherrors.py @@ -0,0 +1,41 @@ +# sherrors.py - shell errors and signals +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +"""Define shell exceptions and error codes. +""" + +class ShellError(Exception): + pass + +class ShellSyntaxError(ShellError): + pass + +class UtilityError(ShellError): + """Raised upon utility syntax error (option or operand error).""" + pass + +class ExpansionError(ShellError): + pass + +class CommandNotFound(ShellError): + """Specified command was not found.""" + pass + +class RedirectionError(ShellError): + pass + +class VarAssignmentError(ShellError): + """Variable assignment error.""" + pass + +class ExitSignal(ShellError): + """Exit signal.""" + pass + +class ReturnSignal(ShellError): + """Exit signal.""" + pass
\ No newline at end of file diff --git a/bitbake/lib/pysh/subprocess_fix.py b/bitbake/lib/pysh/subprocess_fix.py new file mode 100644 index 0000000000..46eca22802 --- /dev/null +++ b/bitbake/lib/pysh/subprocess_fix.py @@ -0,0 +1,77 @@ +# subprocess - Subprocesses with accessible I/O streams +# +# For more information about this module, see PEP 324. +# +# This module should remain compatible with Python 2.2, see PEP 291. +# +# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> +# +# Licensed to PSF under a Contributor Agreement. +# See http://www.python.org/2.4/license for licensing details. + +def list2cmdline(seq): + """ + Translate a sequence of arguments into a command line + string, using the same rules as the MS C runtime: + + 1) Arguments are delimited by white space, which is either a + space or a tab. + + 2) A string surrounded by double quotation marks is + interpreted as a single argument, regardless of white space + contained within. A quoted string can be embedded in an + argument. + + 3) A double quotation mark preceded by a backslash is + interpreted as a literal double quotation mark. + + 4) Backslashes are interpreted literally, unless they + immediately precede a double quotation mark. + + 5) If backslashes immediately precede a double quotation mark, + every pair of backslashes is interpreted as a literal + backslash. If the number of backslashes is odd, the last + backslash escapes the next double quotation mark as + described in rule 3. + """ + + # See + # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp + result = [] + needquote = False + for arg in seq: + bs_buf = [] + + # Add a space to separate this argument from the others + if result: + result.append(' ') + + needquote = (" " in arg) or ("\t" in arg) or ("|" in arg) or arg == "" + if needquote: + result.append('"') + + for c in arg: + if c == '\\': + # Don't know if we need to double yet. + bs_buf.append(c) + elif c == '"': + # Double backspaces. + result.append('\\' * len(bs_buf)*2) + bs_buf = [] + result.append('\\"') + else: + # Normal char + if bs_buf: + result.extend(bs_buf) + bs_buf = [] + result.append(c) + + # Add remaining backspaces, if any. + if bs_buf: + result.extend(bs_buf) + + if needquote: + result.extend(bs_buf) + result.append('"') + + return ''.join(result) |