diff options
| author | Khem Raj <raj.khem@gmail.com> | 2011-02-23 10:48:51 -0800 |
|---|---|---|
| committer | Khem Raj <raj.khem@gmail.com> | 2011-02-23 10:48:51 -0800 |
| commit | 951cbf3f65f347c7a7bbcae193218f9187a15fbf (patch) | |
| tree | 96e9c74551e6931804992f8f49af05f732eb0fff /bitbake/lib/bb/pysh | |
| parent | adbaae2179a6c3746e53f7fbb2ca0939e85a7ea9 (diff) | |
| download | openembedded-core-951cbf3f65f347c7a7bbcae193218f9187a15fbf.tar.gz openembedded-core-951cbf3f65f347c7a7bbcae193218f9187a15fbf.tar.bz2 openembedded-core-951cbf3f65f347c7a7bbcae193218f9187a15fbf.zip | |
bitbake: Remove in-tree version
Bitbake should be used by checking it out from its own repo
Signed-off-by: Khem Raj <raj.khem@gmail.com>
Diffstat (limited to 'bitbake/lib/bb/pysh')
| -rw-r--r-- | bitbake/lib/bb/pysh/__init__.py | 0 | ||||
| -rw-r--r-- | bitbake/lib/bb/pysh/builtin.py | 710 | ||||
| -rw-r--r-- | bitbake/lib/bb/pysh/interp.py | 1367 | ||||
| -rw-r--r-- | bitbake/lib/bb/pysh/lsprof.py | 116 | ||||
| -rw-r--r-- | bitbake/lib/bb/pysh/pysh.py | 167 | ||||
| -rw-r--r-- | bitbake/lib/bb/pysh/pyshlex.py | 888 | ||||
| -rw-r--r-- | bitbake/lib/bb/pysh/pyshyacc.py | 779 | ||||
| -rw-r--r-- | bitbake/lib/bb/pysh/sherrors.py | 41 | ||||
| -rw-r--r-- | bitbake/lib/bb/pysh/subprocess_fix.py | 77 |
9 files changed, 0 insertions, 4145 deletions
diff --git a/bitbake/lib/bb/pysh/__init__.py b/bitbake/lib/bb/pysh/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 --- a/bitbake/lib/bb/pysh/__init__.py +++ /dev/null diff --git a/bitbake/lib/bb/pysh/builtin.py b/bitbake/lib/bb/pysh/builtin.py deleted file mode 100644 index 25ad22eb74..0000000000 --- a/bitbake/lib/bb/pysh/builtin.py +++ /dev/null @@ -1,710 +0,0 @@ -# builtin.py - builtins and utilities definitions for pysh. -# -# Copyright 2007 Patrick Mezard -# -# This software may be used and distributed according to the terms -# of the GNU General Public License, incorporated herein by reference. - -"""Builtin and internal utilities implementations. - -- Beware not to use python interpreter environment as if it were the shell -environment. For instance, commands working directory must be explicitely handled -through env['PWD'] instead of relying on python working directory. -""" -import errno -import optparse -import os -import re -import subprocess -import sys -import time - -def has_subprocess_bug(): - return getattr(subprocess, 'list2cmdline') and \ - ( subprocess.list2cmdline(['']) == '' or \ - subprocess.list2cmdline(['foo|bar']) == 'foo|bar') - -# Detect python bug 1634343: "subprocess swallows empty arguments under win32" -# <http://sourceforge.net/tracker/index.php?func=detail&aid=1634343&group_id=5470&atid=105470> -# Also detect: "[ 1710802 ] subprocess must escape redirection characters under win32" -# <http://sourceforge.net/tracker/index.php?func=detail&aid=1710802&group_id=5470&atid=105470> -if has_subprocess_bug(): - import subprocess_fix - subprocess.list2cmdline = subprocess_fix.list2cmdline - -from sherrors import * - -class NonExitingParser(optparse.OptionParser): - """OptionParser default behaviour upon error is to print the error message and - exit. Raise a utility error instead. - """ - def error(self, msg): - raise UtilityError(msg) - -#------------------------------------------------------------------------------- -# set special builtin -#------------------------------------------------------------------------------- -OPT_SET = NonExitingParser(usage="set - set or unset options and positional parameters") -OPT_SET.add_option( '-f', action='store_true', dest='has_f', default=False, - help='The shell shall disable pathname expansion.') -OPT_SET.add_option('-e', action='store_true', dest='has_e', default=False, - help="""When this option is on, if a simple command fails for any of the \ - reasons listed in Consequences of Shell Errors or returns an exit status \ - value >0, and is not part of the compound list following a while, until, \ - or if keyword, and is not a part of an AND or OR list, and is not a \ - pipeline preceded by the ! reserved word, then the shell shall immediately \ - exit.""") -OPT_SET.add_option('-x', action='store_true', dest='has_x', default=False, - help="""The shell shall write to standard error a trace for each command \ - after it expands the command and before it executes it. It is unspecified \ - whether the command that turns tracing off is traced.""") - -def builtin_set(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - option, args = OPT_SET.parse_args(args) - env = interp.get_env() - - if option.has_f: - env.set_opt('-f') - if option.has_e: - env.set_opt('-e') - if option.has_x: - env.set_opt('-x') - return 0 - -#------------------------------------------------------------------------------- -# shift special builtin -#------------------------------------------------------------------------------- -def builtin_shift(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - params = interp.get_env().get_positional_args() - if args: - try: - n = int(args[0]) - if n > len(params): - raise ValueError() - except ValueError: - return 1 - else: - n = 1 - - params[:n] = [] - interp.get_env().set_positional_args(params) - return 0 - -#------------------------------------------------------------------------------- -# export special builtin -#------------------------------------------------------------------------------- -OPT_EXPORT = NonExitingParser(usage="set - set or unset options and positional parameters") -OPT_EXPORT.add_option('-p', action='store_true', dest='has_p', default=False) - -def builtin_export(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - option, args = OPT_EXPORT.parse_args(args) - if option.has_p: - raise NotImplementedError() - - for arg in args: - try: - name, value = arg.split('=', 1) - except ValueError: - name, value = arg, None - env = interp.get_env().export(name, value) - - return 0 - -#------------------------------------------------------------------------------- -# return special builtin -#------------------------------------------------------------------------------- -def builtin_return(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - res = 0 - if args: - try: - res = int(args[0]) - except ValueError: - res = 0 - if not 0<=res<=255: - res = 0 - - # BUG: should be last executed command exit code - raise ReturnSignal(res) - -#------------------------------------------------------------------------------- -# trap special builtin -#------------------------------------------------------------------------------- -def builtin_trap(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - if len(args) < 2: - stderr.write('trap: usage: trap [[arg] signal_spec ...]\n') - return 2 - - action = args[0] - for sig in args[1:]: - try: - env.traps[sig] = action - except Exception, e: - stderr.write('trap: %s\n' % str(e)) - return 0 - -#------------------------------------------------------------------------------- -# unset special builtin -#------------------------------------------------------------------------------- -OPT_UNSET = NonExitingParser("unset - unset values and attributes of variables and functions") -OPT_UNSET.add_option( '-f', action='store_true', dest='has_f', default=False) -OPT_UNSET.add_option( '-v', action='store_true', dest='has_v', default=False) - -def builtin_unset(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - option, args = OPT_UNSET.parse_args(args) - - status = 0 - env = interp.get_env() - for arg in args: - try: - if option.has_f: - env.remove_function(arg) - else: - del env[arg] - except KeyError: - pass - except VarAssignmentError: - status = 1 - - return status - -#------------------------------------------------------------------------------- -# wait special builtin -#------------------------------------------------------------------------------- -def builtin_wait(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - return interp.wait([int(arg) for arg in args]) - -#------------------------------------------------------------------------------- -# cat utility -#------------------------------------------------------------------------------- -def utility_cat(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - if not args: - args = ['-'] - - status = 0 - for arg in args: - if arg == '-': - data = stdin.read() - else: - path = os.path.join(env['PWD'], arg) - try: - f = file(path, 'rb') - try: - data = f.read() - finally: - f.close() - except IOError, e: - if e.errno != errno.ENOENT: - raise - status = 1 - continue - stdout.write(data) - stdout.flush() - return status - -#------------------------------------------------------------------------------- -# cd utility -#------------------------------------------------------------------------------- -OPT_CD = NonExitingParser("cd - change the working directory") - -def utility_cd(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - option, args = OPT_CD.parse_args(args) - env = interp.get_env() - - directory = None - printdir = False - if not args: - home = env.get('HOME') - if home: - # Unspecified, do nothing - return 0 - else: - directory = home - elif len(args)==1: - directory = args[0] - if directory=='-': - if 'OLDPWD' not in env: - raise UtilityError("OLDPWD not set") - printdir = True - directory = env['OLDPWD'] - else: - raise UtilityError("too many arguments") - - curpath = None - # Absolute directories will be handled correctly by the os.path.join call. - if not directory.startswith('.') and not directory.startswith('..'): - cdpaths = env.get('CDPATH', '.').split(';') - for cdpath in cdpaths: - p = os.path.join(cdpath, directory) - if os.path.isdir(p): - curpath = p - break - - if curpath is None: - curpath = directory - curpath = os.path.join(env['PWD'], directory) - - env['OLDPWD'] = env['PWD'] - env['PWD'] = curpath - if printdir: - stdout.write('%s\n' % curpath) - return 0 - -#------------------------------------------------------------------------------- -# colon utility -#------------------------------------------------------------------------------- -def utility_colon(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - return 0 - -#------------------------------------------------------------------------------- -# echo utility -#------------------------------------------------------------------------------- -def utility_echo(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - # Echo only takes arguments, no options. Use printf if you need fancy stuff. - output = ' '.join(args) + '\n' - stdout.write(output) - stdout.flush() - return 0 - -#------------------------------------------------------------------------------- -# egrep utility -#------------------------------------------------------------------------------- -# egrep is usually a shell script. -# Unfortunately, pysh does not support shell scripts *with arguments* right now, -# so the redirection is implemented here, assuming grep is available. -def utility_egrep(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - return run_command('grep', ['-E'] + args, interp, env, stdin, stdout, - stderr, debugflags) - -#------------------------------------------------------------------------------- -# env utility -#------------------------------------------------------------------------------- -def utility_env(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - if args and args[0]=='-i': - raise NotImplementedError('env: -i option is not implemented') - - i = 0 - for arg in args: - if '=' not in arg: - break - # Update the current environment - name, value = arg.split('=', 1) - env[name] = value - i += 1 - - if args[i:]: - # Find then execute the specified interpreter - utility = env.find_in_path(args[i]) - if not utility: - return 127 - args[i:i+1] = utility - name = args[i] - args = args[i+1:] - try: - return run_command(name, args, interp, env, stdin, stdout, stderr, - debugflags) - except UtilityError: - stderr.write('env: failed to execute %s' % ' '.join([name]+args)) - return 126 - else: - for pair in env.get_variables().iteritems(): - stdout.write('%s=%s\n' % pair) - return 0 - -#------------------------------------------------------------------------------- -# exit utility -#------------------------------------------------------------------------------- -def utility_exit(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - res = None - if args: - try: - res = int(args[0]) - except ValueError: - res = None - if not 0<=res<=255: - res = None - - if res is None: - # BUG: should be last executed command exit code - res = 0 - - raise ExitSignal(res) - -#------------------------------------------------------------------------------- -# fgrep utility -#------------------------------------------------------------------------------- -# see egrep -def utility_fgrep(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - return run_command('grep', ['-F'] + args, interp, env, stdin, stdout, - stderr, debugflags) - -#------------------------------------------------------------------------------- -# gunzip utility -#------------------------------------------------------------------------------- -# see egrep -def utility_gunzip(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - return run_command('gzip', ['-d'] + args, interp, env, stdin, stdout, - stderr, debugflags) - -#------------------------------------------------------------------------------- -# kill utility -#------------------------------------------------------------------------------- -def utility_kill(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - for arg in args: - pid = int(arg) - status = subprocess.call(['pskill', '/T', str(pid)], - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - # pskill is asynchronous, hence the stupid polling loop - while 1: - p = subprocess.Popen(['pslist', str(pid)], - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - output = p.communicate()[0] - if ('process %d was not' % pid) in output: - break - time.sleep(1) - return status - -#------------------------------------------------------------------------------- -# mkdir utility -#------------------------------------------------------------------------------- -OPT_MKDIR = NonExitingParser("mkdir - make directories.") -OPT_MKDIR.add_option('-p', action='store_true', dest='has_p', default=False) - -def utility_mkdir(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - # TODO: implement umask - # TODO: implement proper utility error report - option, args = OPT_MKDIR.parse_args(args) - for arg in args: - path = os.path.join(env['PWD'], arg) - if option.has_p: - try: - os.makedirs(path) - except IOError, e: - if e.errno != errno.EEXIST: - raise - else: - os.mkdir(path) - return 0 - -#------------------------------------------------------------------------------- -# netstat utility -#------------------------------------------------------------------------------- -def utility_netstat(name, args, interp, env, stdin, stdout, stderr, debugflags): - # Do you really expect me to implement netstat ? - # This empty form is enough for Mercurial tests since it's - # supposed to generate nothing upon success. Faking this test - # is not a big deal either. - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - return 0 - -#------------------------------------------------------------------------------- -# pwd utility -#------------------------------------------------------------------------------- -OPT_PWD = NonExitingParser("pwd - return working directory name") -OPT_PWD.add_option('-L', action='store_true', dest='has_L', default=True, - help="""If the PWD environment variable contains an absolute pathname of \ - the current directory that does not contain the filenames dot or dot-dot, \ - pwd shall write this pathname to standard output. Otherwise, the -L option \ - shall behave as the -P option.""") -OPT_PWD.add_option('-P', action='store_true', dest='has_L', default=False, - help="""The absolute pathname written shall not contain filenames that, in \ - the context of the pathname, refer to files of type symbolic link.""") - -def utility_pwd(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - option, args = OPT_PWD.parse_args(args) - stdout.write('%s\n' % env['PWD']) - return 0 - -#------------------------------------------------------------------------------- -# printf utility -#------------------------------------------------------------------------------- -RE_UNESCAPE = re.compile(r'(\\x[a-zA-Z0-9]{2}|\\[0-7]{1,3}|\\.)') - -def utility_printf(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - def replace(m): - assert m.group() - g = m.group()[1:] - if g.startswith('x'): - return chr(int(g[1:], 16)) - if len(g) <= 3 and len([c for c in g if c in '01234567']) == len(g): - # Yay, an octal number - return chr(int(g, 8)) - return { - 'a': '\a', - 'b': '\b', - 'f': '\f', - 'n': '\n', - 'r': '\r', - 't': '\t', - 'v': '\v', - '\\': '\\', - }.get(g) - - # Convert escape sequences - format = re.sub(RE_UNESCAPE, replace, args[0]) - stdout.write(format % tuple(args[1:])) - return 0 - -#------------------------------------------------------------------------------- -# true utility -#------------------------------------------------------------------------------- -def utility_true(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - return 0 - -#------------------------------------------------------------------------------- -# sed utility -#------------------------------------------------------------------------------- -RE_SED = re.compile(r'^s(.).*\1[a-zA-Z]*$') - -# cygwin sed fails with some expressions when they do not end with a single space. -# see unit tests for details. Interestingly, the same expressions works perfectly -# in cygwin shell. -def utility_sed(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - # Scan pattern arguments and append a space if necessary - for i in xrange(len(args)): - if not RE_SED.search(args[i]): - continue - args[i] = args[i] + ' ' - - return run_command(name, args, interp, env, stdin, stdout, - stderr, debugflags) - -#------------------------------------------------------------------------------- -# sleep utility -#------------------------------------------------------------------------------- -def utility_sleep(name, args, interp, env, stdin, stdout, stderr, debugflags): - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - time.sleep(int(args[0])) - return 0 - -#------------------------------------------------------------------------------- -# sort utility -#------------------------------------------------------------------------------- -OPT_SORT = NonExitingParser("sort - sort, merge, or sequence check text files") - -def utility_sort(name, args, interp, env, stdin, stdout, stderr, debugflags): - - def sort(path): - if path == '-': - lines = stdin.readlines() - else: - try: - f = file(path) - try: - lines = f.readlines() - finally: - f.close() - except IOError, e: - stderr.write(str(e) + '\n') - return 1 - - if lines and lines[-1][-1]!='\n': - lines[-1] = lines[-1] + '\n' - return lines - - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - option, args = OPT_SORT.parse_args(args) - alllines = [] - - if len(args)<=0: - args += ['-'] - - # Load all files lines - curdir = os.getcwd() - try: - os.chdir(env['PWD']) - for path in args: - alllines += sort(path) - finally: - os.chdir(curdir) - - alllines.sort() - for line in alllines: - stdout.write(line) - return 0 - -#------------------------------------------------------------------------------- -# hg utility -#------------------------------------------------------------------------------- - -hgcommands = [ - 'add', - 'addremove', - 'commit', 'ci', - 'debugrename', - 'debugwalk', - 'falabala', # Dummy command used in a mercurial test - 'incoming', - 'locate', - 'pull', - 'push', - 'qinit', - 'remove', 'rm', - 'rename', 'mv', - 'revert', - 'showconfig', - 'status', 'st', - 'strip', - ] - -def rewriteslashes(name, args): - # Several hg commands output file paths, rewrite the separators - if len(args) > 1 and name.lower().endswith('python') \ - and args[0].endswith('hg'): - for cmd in hgcommands: - if cmd in args[1:]: - return True - - # svn output contains many paths with OS specific separators. - # Normalize these to unix paths. - base = os.path.basename(name) - if base.startswith('svn'): - return True - - return False - -def rewritehg(output): - if not output: - return output - # Rewrite os specific messages - output = output.replace(': The system cannot find the file specified', - ': No such file or directory') - output = re.sub(': Access is denied.*$', ': Permission denied', output) - output = output.replace(': No connection could be made because the target machine actively refused it', - ': Connection refused') - return output - - -def run_command(name, args, interp, env, stdin, stdout, - stderr, debugflags): - # Execute the command - if 'debug-utility' in debugflags: - print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n') - - hgbin = interp.options().hgbinary - ishg = hgbin and ('hg' in name or args and 'hg' in args[0]) - unixoutput = 'cygwin' in name or ishg - - exec_env = env.get_variables() - try: - # BUG: comparing file descriptor is clearly not a reliable way to tell - # whether they point on the same underlying object. But in pysh limited - # scope this is usually right, we do not expect complicated redirections - # besides usual 2>&1. - # Still there is one case we have but cannot deal with is when stdout - # and stderr are redirected *by pysh caller*. This the reason for the - # --redirect pysh() option. - # Now, we want to know they are the same because we sometimes need to - # transform the command output, mostly remove CR-LF to ensure that - # command output is unix-like. Cygwin utilies are a special case because - # they explicitely set their output streams to binary mode, so we have - # nothing to do. For all others commands, we have to guess whether they - # are sending text data, in which case the transformation must be done. - # Again, the NUL character test is unreliable but should be enough for - # hg tests. - redirected = stdout.fileno()==stderr.fileno() - if not redirected: - p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env, - stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env, - stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out, err = p.communicate() - except WindowsError, e: - raise UtilityError(str(e)) - - if not unixoutput: - def encode(s): - if '\0' in s: - return s - return s.replace('\r\n', '\n') - else: - encode = lambda s: s - - if rewriteslashes(name, args): - encode1_ = encode - def encode(s): - s = encode1_(s) - s = s.replace('\\\\', '\\') - s = s.replace('\\', '/') - return s - - if ishg: - encode2_ = encode - def encode(s): - return rewritehg(encode2_(s)) - - stdout.write(encode(out)) - if not redirected: - stderr.write(encode(err)) - return p.returncode - diff --git a/bitbake/lib/bb/pysh/interp.py b/bitbake/lib/bb/pysh/interp.py deleted file mode 100644 index efe5181e1e..0000000000 --- a/bitbake/lib/bb/pysh/interp.py +++ /dev/null @@ -1,1367 +0,0 @@ -# interp.py - shell interpreter for pysh. -# -# Copyright 2007 Patrick Mezard -# -# This software may be used and distributed according to the terms -# of the GNU General Public License, incorporated herein by reference. - -"""Implement the shell interpreter. - -Most references are made to "The Open Group Base Specifications Issue 6". -<http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html> -""" -# TODO: document the fact input streams must implement fileno() so Popen will work correctly. -# it requires non-stdin stream to be implemented as files. Still to be tested... -# DOC: pathsep is used in PATH instead of ':'. Clearly, there are path syntax issues here. -# TODO: stop command execution upon error. -# TODO: sort out the filename/io_number mess. It should be possible to use filenames only. -# TODO: review subshell implementation -# TODO: test environment cloning for non-special builtins -# TODO: set -x should not rebuild commands from tokens, assignments/redirections are lost -# TODO: unit test for variable assignment -# TODO: test error management wrt error type/utility type -# TODO: test for binary output everywhere -# BUG: debug-parsing does not pass log file to PLY. Maybe a PLY upgrade is necessary. -import base64 -import cPickle as pickle -import errno -import glob -import os -import re -import subprocess -import sys -import tempfile - -try: - s = set() - del s -except NameError: - from Set import Set as set - -import builtin -from sherrors import * -import pyshlex -import pyshyacc - -def mappend(func, *args, **kargs): - """Like map but assume func returns a list. Returned lists are merged into - a single one. - """ - return reduce(lambda a,b: a+b, map(func, *args, **kargs), []) - -class FileWrapper: - """File object wrapper to ease debugging. - - Allow mode checking and implement file duplication through a simple - reference counting scheme. Not sure the latter is really useful since - only real file descriptors can be used. - """ - def __init__(self, mode, file, close=True): - if mode not in ('r', 'w', 'a'): - raise IOError('invalid mode: %s' % mode) - self._mode = mode - self._close = close - if isinstance(file, FileWrapper): - if file._refcount[0] <= 0: - raise IOError(0, 'Error') - self._refcount = file._refcount - self._refcount[0] += 1 - self._file = file._file - else: - self._refcount = [1] - self._file = file - - def dup(self): - return FileWrapper(self._mode, self, self._close) - - def fileno(self): - """fileno() should be only necessary for input streams.""" - return self._file.fileno() - - def read(self, size=-1): - if self._mode!='r': - raise IOError(0, 'Error') - return self._file.read(size) - - def readlines(self, *args, **kwargs): - return self._file.readlines(*args, **kwargs) - - def write(self, s): - if self._mode not in ('w', 'a'): - raise IOError(0, 'Error') - return self._file.write(s) - - def flush(self): - self._file.flush() - - def close(self): - if not self._refcount: - return - assert self._refcount[0] > 0 - - self._refcount[0] -= 1 - if self._refcount[0] == 0: - self._mode = 'c' - if self._close: - self._file.close() - self._refcount = None - - def mode(self): - return self._mode - - def __getattr__(self, name): - if name == 'name': - self.name = getattr(self._file, name) - return self.name - else: - raise AttributeError(name) - - def __del__(self): - self.close() - - -def win32_open_devnull(mode): - return open('NUL', mode) - - -class Redirections: - """Stores open files and their mapping to pseudo-sh file descriptor. - """ - # BUG: redirections are not handled correctly: 1>&3 2>&3 3>&4 does - # not make 1 to redirect to 4 - def __init__(self, stdin=None, stdout=None, stderr=None): - self._descriptors = {} - if stdin is not None: - self._add_descriptor(0, stdin) - if stdout is not None: - self._add_descriptor(1, stdout) - if stderr is not None: - self._add_descriptor(2, stderr) - - def add_here_document(self, interp, name, content, io_number=None): - if io_number is None: - io_number = 0 - - if name==pyshlex.unquote_wordtree(name): - content = interp.expand_here_document(('TOKEN', content)) - - # Write document content in a temporary file - tmp = tempfile.TemporaryFile() - try: - tmp.write(content) - tmp.flush() - tmp.seek(0) - self._add_descriptor(io_number, FileWrapper('r', tmp)) - except: - tmp.close() - raise - - def add(self, interp, op, filename, io_number=None): - if op not in ('<', '>', '>|', '>>', '>&'): - # TODO: add descriptor duplication and here_documents - raise RedirectionError('Unsupported redirection operator "%s"' % op) - - if io_number is not None: - io_number = int(io_number) - - if (op == '>&' and filename.isdigit()) or filename=='-': - # No expansion for file descriptors, quote them if you want a filename - fullname = filename - else: - if filename.startswith('/'): - # TODO: win32 kludge - if filename=='/dev/null': - fullname = 'NUL' - else: - # TODO: handle absolute pathnames, they are unlikely to exist on the - # current platform (win32 for instance). - raise NotImplementedError() - else: - fullname = interp.expand_redirection(('TOKEN', filename)) - if not fullname: - raise RedirectionError('%s: ambiguous redirect' % filename) - # Build absolute path based on PWD - fullname = os.path.join(interp.get_env()['PWD'], fullname) - - if op=='<': - return self._add_input_redirection(interp, fullname, io_number) - elif op in ('>', '>|'): - clobber = ('>|'==op) - return self._add_output_redirection(interp, fullname, io_number, clobber) - elif op=='>>': - return self._add_output_appending(interp, fullname, io_number) - elif op=='>&': - return self._dup_output_descriptor(fullname, io_number) - - def close(self): - if self._descriptors is not None: - for desc in self._descriptors.itervalues(): - desc.flush() - desc.close() - self._descriptors = None - - def stdin(self): - return self._descriptors[0] - - def stdout(self): - return self._descriptors[1] - - def stderr(self): - return self._descriptors[2] - - def clone(self): - clone = Redirections() - for desc, fileobj in self._descriptors.iteritems(): - clone._descriptors[desc] = fileobj.dup() - return clone - - def _add_output_redirection(self, interp, filename, io_number, clobber): - if io_number is None: - # io_number default to standard output - io_number = 1 - - if not clobber and interp.get_env().has_opt('-C') and os.path.isfile(filename): - # File already exist in no-clobber mode, bail out - raise RedirectionError('File "%s" already exists' % filename) - - # Open and register - self._add_file_descriptor(io_number, filename, 'w') - - def _add_output_appending(self, interp, filename, io_number): - if io_number is None: - io_number = 1 - self._add_file_descriptor(io_number, filename, 'a') - - def _add_input_redirection(self, interp, filename, io_number): - if io_number is None: - io_number = 0 - self._add_file_descriptor(io_number, filename, 'r') - - def _add_file_descriptor(self, io_number, filename, mode): - try: - if filename.startswith('/'): - if filename=='/dev/null': - f = win32_open_devnull(mode+'b') - else: - # TODO: handle absolute pathnames, they are unlikely to exist on the - # current platform (win32 for instance). - raise NotImplementedError('cannot open absolute path %s' % repr(filename)) - else: - f = file(filename, mode+'b') - except IOError, e: - raise RedirectionError(str(e)) - - wrapper = None - try: - wrapper = FileWrapper(mode, f) - f = None - self._add_descriptor(io_number, wrapper) - except: - if f: f.close() - if wrapper: wrapper.close() - raise - - def _dup_output_descriptor(self, source_fd, dest_fd): - if source_fd is None: - source_fd = 1 - self._dup_file_descriptor(source_fd, dest_fd, 'w') - - def _dup_file_descriptor(self, source_fd, dest_fd, mode): - source_fd = int(source_fd) - if source_fd not in self._descriptors: - raise RedirectionError('"%s" is not a valid file descriptor' % str(source_fd)) - source = self._descriptors[source_fd] - - if source.mode()!=mode: - raise RedirectionError('Descriptor %s cannot be duplicated in mode "%s"' % (str(source), mode)) - - if dest_fd=='-': - # Close the source descriptor - del self._descriptors[source_fd] - source.close() - else: - dest_fd = int(dest_fd) - if dest_fd not in self._descriptors: - raise RedirectionError('Cannot replace file descriptor %s' % str(dest_fd)) - - dest = self._descriptors[dest_fd] - if dest.mode()!=mode: - raise RedirectionError('Descriptor %s cannot be cannot be redirected in mode "%s"' % (str(dest), mode)) - - self._descriptors[dest_fd] = source.dup() - dest.close() - - def _add_descriptor(self, io_number, file): - io_number = int(io_number) - - if io_number in self._descriptors: - # Close the current descriptor - d = self._descriptors[io_number] - del self._descriptors[io_number] - d.close() - - self._descriptors[io_number] = file - - def __str__(self): - names = [('%d=%r' % (k, getattr(v, 'name', None))) for k,v - in self._descriptors.iteritems()] - names = ','.join(names) - return 'Redirections(%s)' % names - - def __del__(self): - self.close() - -def cygwin_to_windows_path(path): - """Turn /cygdrive/c/foo into c:/foo, or return path if it - is not a cygwin path. - """ - if not path.startswith('/cygdrive/'): - return path - path = path[len('/cygdrive/'):] - path = path[:1] + ':' + path[1:] - return path - -def win32_to_unix_path(path): - if path is not None: - path = path.replace('\\', '/') - return path - -_RE_SHEBANG = re.compile(r'^\#!\s?([^\s]+)(?:\s([^\s]+))?') -_SHEBANG_CMDS = { - '/usr/bin/env': 'env', - '/bin/sh': 'pysh', - 'python': 'python', -} - -def resolve_shebang(path, ignoreshell=False): - """Return a list of arguments as shebang interpreter call or an empty list - if path does not refer to an executable script. - See <http://www.opengroup.org/austin/docs/austin_51r2.txt>. - - ignoreshell - set to True to ignore sh shebangs. Return an empty list instead. - """ - try: - f = file(path) - try: - # At most 80 characters in the first line - header = f.read(80).splitlines()[0] - finally: - f.close() - - m = _RE_SHEBANG.search(header) - if not m: - return [] - cmd, arg = m.group(1,2) - if os.path.isfile(cmd): - # Keep this one, the hg script for instance contains a weird windows - # shebang referencing the current python install. - cmdfile = os.path.basename(cmd).lower() - if cmdfile == 'python.exe': - cmd = 'python' - pass - elif cmd not in _SHEBANG_CMDS: - raise CommandNotFound('Unknown interpreter "%s" referenced in '\ - 'shebang' % header) - cmd = _SHEBANG_CMDS.get(cmd) - if cmd is None or (ignoreshell and cmd == 'pysh'): - return [] - if arg is None: - return [cmd, win32_to_unix_path(path)] - return [cmd, arg, win32_to_unix_path(path)] - except IOError, e: - if e.errno!=errno.ENOENT and \ - (e.errno!=errno.EPERM and not os.path.isdir(path)): # Opening a directory raises EPERM - raise - return [] - -def win32_find_in_path(name, path): - if isinstance(path, str): - path = path.split(os.pathsep) - - exts = os.environ.get('PATHEXT', '').lower().split(os.pathsep) - for p in path: - p_name = os.path.join(p, name) - - prefix = resolve_shebang(p_name) - if prefix: - return prefix - - for ext in exts: - p_name_ext = p_name + ext - if os.path.exists(p_name_ext): - return [win32_to_unix_path(p_name_ext)] - return [] - -class Traps(dict): - def __setitem__(self, key, value): - if key not in ('EXIT',): - raise NotImplementedError() - super(Traps, self).__setitem__(key, value) - -# IFS white spaces character class -_IFS_WHITESPACES = (' ', '\t', '\n') - -class Environment: - """Environment holds environment variables, export table, function - definitions and whatever is defined in 2.12 "Shell Execution Environment", - redirection excepted. - """ - def __init__(self, pwd): - self._opt = set() #Shell options - - self._functions = {} - self._env = {'?': '0', '#': '0'} - self._exported = set([ - 'HOME', 'IFS', 'PATH' - ]) - - # Set environment vars with side-effects - self._ifs_ws = None # Set of IFS whitespace characters - self._ifs_re = None # Regular expression used to split between words using IFS classes - self['IFS'] = ''.join(_IFS_WHITESPACES) #Default environment values - self['PWD'] = pwd - self.traps = Traps() - - def clone(self, subshell=False): - env = Environment(self['PWD']) - env._opt = set(self._opt) - for k,v in self.get_variables().iteritems(): - if k in self._exported: - env.export(k,v) - elif subshell: - env[k] = v - - if subshell: - env._functions = dict(self._functions) - - return env - - def __getitem__(self, key): - if key in ('@', '*', '-', '$'): - raise NotImplementedError('%s is not implemented' % repr(key)) - return self._env[key] - - def get(self, key, defval=None): - try: - return self[key] - except KeyError: - return defval - - def __setitem__(self, key, value): - if key=='IFS': - # Update the whitespace/non-whitespace classes - self._update_ifs(value) - elif key=='PWD': - pwd = os.path.abspath(value) - if not os.path.isdir(pwd): - raise VarAssignmentError('Invalid directory %s' % value) - value = pwd - elif key in ('?', '!'): - value = str(int(value)) - self._env[key] = value - - def __delitem__(self, key): - if key in ('IFS', 'PWD', '?'): - raise VarAssignmentError('%s cannot be unset' % key) - del self._env[key] - - def __contains__(self, item): - return item in self._env - - def set_positional_args(self, args): - """Set the content of 'args' as positional argument from 1 to len(args). - Return previous argument as a list of strings. - """ - # Save and remove previous arguments - prevargs = [] - for i in xrange(int(self._env['#'])): - i = str(i+1) - prevargs.append(self._env[i]) - del self._env[i] - self._env['#'] = '0' - - #Set new ones - for i,arg in enumerate(args): - self._env[str(i+1)] = str(arg) - self._env['#'] = str(len(args)) - - return prevargs - - def get_positional_args(self): - return [self._env[str(i+1)] for i in xrange(int(self._env['#']))] - - def get_variables(self): - return dict(self._env) - - def export(self, key, value=None): - if value is not None: - self[key] = value - self._exported.add(key) - - def get_exported(self): - return [(k,self._env.get(k)) for k in self._exported] - - def split_fields(self, word): - if not self._ifs_ws or not word: - return [word] - return re.split(self._ifs_re, word) - - def _update_ifs(self, value): - """Update the split_fields related variables when IFS character set is - changed. - """ - # TODO: handle NULL IFS - - # Separate characters in whitespace and non-whitespace - chars = set(value) - ws = [c for c in chars if c in _IFS_WHITESPACES] - nws = [c for c in chars if c not in _IFS_WHITESPACES] - - # Keep whitespaces in a string for left and right stripping - self._ifs_ws = ''.join(ws) - - # Build a regexp to split fields - trailing = '[' + ''.join([re.escape(c) for c in ws]) + ']' - if nws: - # First, the single non-whitespace occurence. - nws = '[' + ''.join([re.escape(c) for c in nws]) + ']' - nws = '(?:' + trailing + '*' + nws + trailing + '*' + '|' + trailing + '+)' - else: - # Then mix all parts with quantifiers - nws = trailing + '+' - self._ifs_re = re.compile(nws) - - def has_opt(self, opt, val=None): - return (opt, val) in self._opt - - def set_opt(self, opt, val=None): - self._opt.add((opt, val)) - - def find_in_path(self, name, pwd=False): - path = self._env.get('PATH', '').split(os.pathsep) - if pwd: - path[:0] = [self['PWD']] - if os.name == 'nt': - return win32_find_in_path(name, self._env.get('PATH', '')) - else: - raise NotImplementedError() - - def define_function(self, name, body): - if not is_name(name): - raise ShellSyntaxError('%s is not a valid function name' % repr(name)) - self._functions[name] = body - - def remove_function(self, name): - del self._functions[name] - - def is_function(self, name): - return name in self._functions - - def get_function(self, name): - return self._functions.get(name) - - -name_charset = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_' -name_charset = dict(zip(name_charset,name_charset)) - -def match_name(s): - """Return the length in characters of the longest prefix made of name - allowed characters in s. - """ - for i,c in enumerate(s): - if c not in name_charset: - return s[:i] - return s - -def is_name(s): - return len([c for c in s if c not in name_charset])<=0 - -def is_special_param(c): - return len(c)==1 and c in ('@','*','#','?','-','$','!','0') - -def utility_not_implemented(name, *args, **kwargs): - raise NotImplementedError('%s utility is not implemented' % name) - - -class Utility: - """Define utilities properties: - func -- utility callable. See builtin module for utility samples. - is_special -- see XCU 2.8. - """ - def __init__(self, func, is_special=0): - self.func = func - self.is_special = bool(is_special) - - -def encodeargs(args): - def encodearg(s): - lines = base64.encodestring(s) - lines = [l.splitlines()[0] for l in lines] - return ''.join(lines) - - s = pickle.dumps(args) - return encodearg(s) - -def decodeargs(s): - s = base64.decodestring(s) - return pickle.loads(s) - - -class GlobError(Exception): - pass - -class Options: - def __init__(self): - # True if Mercurial operates with binary streams - self.hgbinary = True - -class Interpreter: - # Implementation is very basic: the execute() method just makes a DFS on the - # AST and execute nodes one by one. Nodes are tuple (name,obj) where name - # is a string identifier and obj the AST element returned by the parser. - # - # Handler are named after the node identifiers. - # TODO: check node names and remove the switch in execute with some - # dynamic getattr() call to find node handlers. - """Shell interpreter. - - The following debugging flags can be passed: - debug-parsing - enable PLY debugging. - debug-tree - print the generated AST. - debug-cmd - trace command execution before word expansion, plus exit status. - debug-utility - trace utility execution. - """ - - # List supported commands. - COMMANDS = { - 'cat': Utility(builtin.utility_cat,), - 'cd': Utility(builtin.utility_cd,), - ':': Utility(builtin.utility_colon,), - 'echo': Utility(builtin.utility_echo), - 'env': Utility(builtin.utility_env), - 'exit': Utility(builtin.utility_exit), - 'export': Utility(builtin.builtin_export, is_special=1), - 'egrep': Utility(builtin.utility_egrep), - 'fgrep': Utility(builtin.utility_fgrep), - 'gunzip': Utility(builtin.utility_gunzip), - 'kill': Utility(builtin.utility_kill), - 'mkdir': Utility(builtin.utility_mkdir), - 'netstat': Utility(builtin.utility_netstat), - 'printf': Utility(builtin.utility_printf), - 'pwd': Utility(builtin.utility_pwd), - 'return': Utility(builtin.builtin_return, is_special=1), - 'sed': Utility(builtin.utility_sed,), - 'set': Utility(builtin.builtin_set,), - 'shift': Utility(builtin.builtin_shift,), - 'sleep': Utility(builtin.utility_sleep,), - 'sort': Utility(builtin.utility_sort,), - 'trap': Utility(builtin.builtin_trap, is_special=1), - 'true': Utility(builtin.utility_true), - 'unset': Utility(builtin.builtin_unset, is_special=1), - 'wait': Utility(builtin.builtin_wait, is_special=1), - } - - def __init__(self, pwd, debugflags = [], env=None, redirs=None, stdin=None, - stdout=None, stderr=None, opts=Options()): - self._env = env - if self._env is None: - self._env = Environment(pwd) - self._children = {} - - self._redirs = redirs - self._close_redirs = False - - if self._redirs is None: - if stdin is None: - stdin = sys.stdin - if stdout is None: - stdout = sys.stdout - if stderr is None: - stderr = sys.stderr - stdin = FileWrapper('r', stdin, False) - stdout = FileWrapper('w', stdout, False) - stderr = FileWrapper('w', stderr, False) - self._redirs = Redirections(stdin, stdout, stderr) - self._close_redirs = True - - self._debugflags = list(debugflags) - self._logfile = sys.stderr - self._options = opts - - def close(self): - """Must be called when the interpreter is no longer used.""" - script = self._env.traps.get('EXIT') - if script: - try: - self.execute_script(script=script) - except: - pass - - if self._redirs is not None and self._close_redirs: - self._redirs.close() - self._redirs = None - - def log(self, s): - self._logfile.write(s) - self._logfile.flush() - - def __getitem__(self, key): - return self._env[key] - - def __setitem__(self, key, value): - self._env[key] = value - - def options(self): - return self._options - - def redirect(self, redirs, ios): - def add_redir(io): - if isinstance(io, pyshyacc.IORedirect): - redirs.add(self, io.op, io.filename, io.io_number) - else: - redirs.add_here_document(self, io.name, io.content, io.io_number) - - map(add_redir, ios) - return redirs - - def execute_script(self, script=None, ast=None, sourced=False, - scriptpath=None): - """If script is not None, parse the input. Otherwise takes the supplied - AST. Then execute the AST. - Return the script exit status. - """ - try: - if scriptpath is not None: - self._env['0'] = os.path.abspath(scriptpath) - - if script is not None: - debug_parsing = ('debug-parsing' in self._debugflags) - cmds, script = pyshyacc.parse(script, True, debug_parsing) - if 'debug-tree' in self._debugflags: - pyshyacc.print_commands(cmds, self._logfile) - self._logfile.flush() - else: - cmds, script = ast, '' - - status = 0 - for cmd in cmds: - try: - status = self.execute(cmd) - except ExitSignal, e: - if sourced: - raise - status = int(e.args[0]) - return status - except ShellError: - self._env['?'] = 1 - raise - if 'debug-utility' in self._debugflags or 'debug-cmd' in self._debugflags: - self.log('returncode ' + str(status)+ '\n') - return status - except CommandNotFound, e: - print >>self._redirs.stderr, str(e) - self._redirs.stderr.flush() - # Command not found by non-interactive shell - # return 127 - raise - except RedirectionError, e: - # TODO: should be handled depending on the utility status - print >>self._redirs.stderr, str(e) - self._redirs.stderr.flush() - # Command not found by non-interactive shell - # return 127 - raise - - def dotcommand(self, env, args): - if len(args) < 1: - raise ShellError('. expects at least one argument') - path = args[0] - if '/' not in path: - found = env.find_in_path(args[0], True) - if found: - path = found[0] - script = file(path).read() - return self.execute_script(script=script, sourced=True) - - def execute(self, token, redirs=None): - """Execute and AST subtree with supplied redirections overriding default - interpreter ones. - Return the exit status. - """ - if not token: - return 0 - - if redirs is None: - redirs = self._redirs - - if isinstance(token, list): - # Commands sequence - res = 0 - for t in token: - res = self.execute(t, redirs) - return res - - type, value = token - status = 0 - if type=='simple_command': - redirs_copy = redirs.clone() - try: - # TODO: define and handle command return values - # TODO: implement set -e - status = self._execute_simple_command(value, redirs_copy) - finally: - redirs_copy.close() - elif type=='pipeline': - status = self._execute_pipeline(value, redirs) - elif type=='and_or': - status = self._execute_and_or(value, redirs) - elif type=='for_clause': - status = self._execute_for_clause(value, redirs) - elif type=='while_clause': - status = self._execute_while_clause(value, redirs) - elif type=='function_definition': - status = self._execute_function_definition(value, redirs) - elif type=='brace_group': - status = self._execute_brace_group(value, redirs) - elif type=='if_clause': - status = self._execute_if_clause(value, redirs) - elif type=='subshell': - status = self.subshell(ast=value.cmds, redirs=redirs) - elif type=='async': - status = self._asynclist(value) - elif type=='redirect_list': - redirs_copy = self.redirect(redirs.clone(), value.redirs) - try: - status = self.execute(value.cmd, redirs_copy) - finally: - redirs_copy.close() - else: - raise NotImplementedError('Unsupported token type ' + type) - - if status < 0: - status = 255 - return status - - def _execute_if_clause(self, if_clause, redirs): - cond_status = self.execute(if_clause.cond, redirs) - if cond_status==0: - return self.execute(if_clause.if_cmds, redirs) - else: - return self.execute(if_clause.else_cmds, redirs) - - def _execute_brace_group(self, group, redirs): - status = 0 - for cmd in group.cmds: - status = self.execute(cmd, redirs) - return status - - def _execute_function_definition(self, fundef, redirs): - self._env.define_function(fundef.name, fundef.body) - return 0 - - def _execute_while_clause(self, while_clause, redirs): - status = 0 - while 1: - cond_status = 0 - for cond in while_clause.condition: - cond_status = self.execute(cond, redirs) - - if cond_status: - break - - for cmd in while_clause.cmds: - status = self.execute(cmd, redirs) - - return status - - def _execute_for_clause(self, for_clause, redirs): - if not is_name(for_clause.name): - raise ShellSyntaxError('%s is not a valid name' % repr(for_clause.name)) - items = mappend(self.expand_token, for_clause.items) - - status = 0 - for item in items: - self._env[for_clause.name] = item - for cmd in for_clause.cmds: - status = self.execute(cmd, redirs) - return status - - def _execute_and_or(self, or_and, redirs): - res = self.execute(or_and.left, redirs) - if (or_and.op=='&&' and res==0) or (or_and.op!='&&' and res!=0): - res = self.execute(or_and.right, redirs) - return res - - def _execute_pipeline(self, pipeline, redirs): - if len(pipeline.commands)==1: - status = self.execute(pipeline.commands[0], redirs) - else: - # Execute all commands one after the other - status = 0 - inpath, outpath = None, None - try: - # Commands inputs and outputs cannot really be plugged as done - # by a real shell. Run commands sequentially and chain their - # input/output throught temporary files. - tmpfd, inpath = tempfile.mkstemp() - os.close(tmpfd) - tmpfd, outpath = tempfile.mkstemp() - os.close(tmpfd) - - inpath = win32_to_unix_path(inpath) - outpath = win32_to_unix_path(outpath) - - for i, cmd in enumerate(pipeline.commands): - call_redirs = redirs.clone() - try: - if i!=0: - call_redirs.add(self, '<', inpath) - if i!=len(pipeline.commands)-1: - call_redirs.add(self, '>', outpath) - - status = self.execute(cmd, call_redirs) - - # Chain inputs/outputs - inpath, outpath = outpath, inpath - finally: - call_redirs.close() - finally: - if inpath: os.remove(inpath) - if outpath: os.remove(outpath) - - if pipeline.reverse_status: - status = int(not status) - self._env['?'] = status - return status - - def _execute_function(self, name, args, interp, env, stdin, stdout, stderr, *others): - assert interp is self - - func = env.get_function(name) - #Set positional parameters - prevargs = None - try: - prevargs = env.set_positional_args(args) - try: - redirs = Redirections(stdin.dup(), stdout.dup(), stderr.dup()) - try: - status = self.execute(func, redirs) - finally: - redirs.close() - except ReturnSignal, e: - status = int(e.args[0]) - env['?'] = status - return status - finally: - #Reset positional parameters - if prevargs is not None: - env.set_positional_args(prevargs) - - def _execute_simple_command(self, token, redirs): - """Can raise ReturnSignal when return builtin is called, ExitSignal when - exit is called, and other shell exceptions upon builtin failures. - """ - debug_command = 'debug-cmd' in self._debugflags - if debug_command: - self.log('word' + repr(token.words) + '\n') - self.log('assigns' + repr(token.assigns) + '\n') - self.log('redirs' + repr(token.redirs) + '\n') - - is_special = None - env = self._env - - try: - # Word expansion - args = [] - for word in token.words: - args += self.expand_token(word) - if is_special is None and args: - is_special = env.is_function(args[0]) or \ - (args[0] in self.COMMANDS and self.COMMANDS[args[0]].is_special) - - if debug_command: - self.log('_execute_simple_command' + str(args) + '\n') - - if not args: - # Redirections happen is a subshell - redirs = redirs.clone() - elif not is_special: - env = self._env.clone() - - # Redirections - self.redirect(redirs, token.redirs) - - # Variables assignments - res = 0 - for type,(k,v) in token.assigns: - status, expanded = self.expand_variable((k,v)) - if status is not None: - res = status - if args: - env.export(k, expanded) - else: - env[k] = expanded - - if args and args[0] in ('.', 'source'): - res = self.dotcommand(env, args[1:]) - elif args: - if args[0] in self.COMMANDS: - command = self.COMMANDS[args[0]] - elif env.is_function(args[0]): - command = Utility(self._execute_function, is_special=True) - else: - if not '/' in args[0].replace('\\', '/'): - cmd = env.find_in_path(args[0]) - if not cmd: - # TODO: test error code on unknown command => 127 - raise CommandNotFound('Unknown command: "%s"' % args[0]) - else: - # Handle commands like '/cygdrive/c/foo.bat' - cmd = cygwin_to_windows_path(args[0]) - if not os.path.exists(cmd): - raise CommandNotFound('%s: No such file or directory' % args[0]) - shebang = resolve_shebang(cmd) - if shebang: - cmd = shebang - else: - cmd = [cmd] - args[0:1] = cmd - command = Utility(builtin.run_command) - - # Command execution - if 'debug-cmd' in self._debugflags: - self.log('redirections ' + str(redirs) + '\n') - - res = command.func(args[0], args[1:], self, env, - redirs.stdin(), redirs.stdout(), - redirs.stderr(), self._debugflags) - - if self._env.has_opt('-x'): - # Trace command execution in shell environment - # BUG: would be hard to reproduce a real shell behaviour since - # the AST is not annotated with source lines/tokens. - self._redirs.stdout().write(' '.join(args)) - - except ReturnSignal: - raise - except ShellError, e: - if is_special or isinstance(e, (ExitSignal, - ShellSyntaxError, ExpansionError)): - raise e - self._redirs.stderr().write(str(e)+'\n') - return 1 - - return res - - def expand_token(self, word): - """Expand a word as specified in [2.6 Word Expansions]. Return the list - of expanded words. - """ - status, wtrees = self._expand_word(word) - return map(pyshlex.wordtree_as_string, wtrees) - - def expand_variable(self, word): - """Return a status code (or None if no command expansion occurred) - and a single word. - """ - status, wtrees = self._expand_word(word, pathname=False, split=False) - words = map(pyshlex.wordtree_as_string, wtrees) - assert len(words)==1 - return status, words[0] - - def expand_here_document(self, word): - """Return the expanded document as a single word. The here document is - assumed to be unquoted. - """ - status, wtrees = self._expand_word(word, pathname=False, - split=False, here_document=True) - words = map(pyshlex.wordtree_as_string, wtrees) - assert len(words)==1 - return words[0] - - def expand_redirection(self, word): - """Return a single word.""" - return self.expand_variable(word)[1] - - def get_env(self): - return self._env - - def _expand_word(self, token, pathname=True, split=True, here_document=False): - wtree = pyshlex.make_wordtree(token[1], here_document=here_document) - - # TODO: implement tilde expansion - def expand(wtree): - """Return a pseudo wordtree: the tree or its subelements can be empty - lists when no value result from the expansion. - """ - status = None - for part in wtree: - if not isinstance(part, list): - continue - if part[0]in ("'", '\\'): - continue - elif part[0] in ('`', '$('): - status, result = self._expand_command(part) - part[:] = result - elif part[0] in ('$', '${'): - part[:] = self._expand_parameter(part, wtree[0]=='"', split) - elif part[0] in ('', '"'): - status, result = expand(part) - part[:] = result - else: - raise NotImplementedError('%s expansion is not implemented' - % part[0]) - # [] is returned when an expansion result in no-field, - # like an empty $@ - wtree = [p for p in wtree if p != []] - if len(wtree) < 3: - return status, [] - return status, wtree - - status, wtree = expand(wtree) - if len(wtree) == 0: - return status, wtree - wtree = pyshlex.normalize_wordtree(wtree) - - if split: - wtrees = self._split_fields(wtree) - else: - wtrees = [wtree] - - if pathname: - wtrees = mappend(self._expand_pathname, wtrees) - - wtrees = map(self._remove_quotes, wtrees) - return status, wtrees - - def _expand_command(self, wtree): - # BUG: there is something to do with backslashes and quoted - # characters here - command = pyshlex.wordtree_as_string(wtree[1:-1]) - status, output = self.subshell_output(command) - return status, ['', output, ''] - - def _expand_parameter(self, wtree, quoted=False, split=False): - """Return a valid wtree or an empty list when no parameter results.""" - # Get the parameter name - # TODO: implement weird expansion rules with ':' - name = pyshlex.wordtree_as_string(wtree[1:-1]) - if not is_name(name) and not is_special_param(name): - raise ExpansionError('Bad substitution "%s"' % name) - # TODO: implement special parameters - if name in ('@', '*'): - args = self._env.get_positional_args() - if len(args) == 0: - return [] - if len(args)<2: - return ['', ''.join(args), ''] - - sep = self._env.get('IFS', '')[:1] - if split and quoted and name=='@': - # Introduce a new token to tell the caller that these parameters - # cause a split as specified in 2.5.2 - return ['@'] + args + [''] - else: - return ['', sep.join(args), ''] - - return ['', self._env.get(name, ''), ''] - - def _split_fields(self, wtree): - def is_empty(split): - return split==['', '', ''] - - def split_positional(quoted): - # Return a list of wtree split according positional parameters rules. - # All remaining '@' groups are removed. - assert quoted[0]=='"' - - splits = [[]] - for part in quoted: - if not isinstance(part, list) or part[0]!='@': - splits[-1].append(part) - else: - # Empty or single argument list were dealt with already - assert len(part)>3 - # First argument must join with the beginning part of the original word - splits[-1].append(part[1]) - # Create double-quotes expressions for every argument after the first - for arg in part[2:-1]: - splits[-1].append('"') - splits.append(['"', arg]) - return splits - - # At this point, all expansions but pathnames have occured. Only quoted - # and positional sequences remain. Thus, all candidates for field splitting - # are in the tree root, or are positional splits ('@') and lie in root - # children. - if not wtree or wtree[0] not in ('', '"'): - # The whole token is quoted or empty, nothing to split - return [wtree] - - if wtree[0]=='"': - wtree = ['', wtree, ''] - - result = [['', '']] - for part in wtree[1:-1]: - if isinstance(part, list): - if part[0]=='"': - splits = split_positional(part) - if len(splits)<=1: - result[-1] += [part, ''] - else: - # Terminate the current split - result[-1] += [splits[0], ''] - result += splits[1:-1] - # Create a new split - result += [['', splits[-1], '']] - else: - result[-1] += [part, ''] - else: - splits = self._env.split_fields(part) - if len(splits)<=1: - # No split - result[-1][-1] += part - else: - # Terminate the current resulting part and create a new one - result[-1][-1] += splits[0] - result[-1].append('') - result += [['', r, ''] for r in splits[1:-1]] - result += [['', splits[-1]]] - result[-1].append('') - - # Leading and trailing empty groups come from leading/trailing blanks - if result and is_empty(result[-1]): - result[-1:] = [] - if result and is_empty(result[0]): - result[:1] = [] - return result - - def _expand_pathname(self, wtree): - """See [2.6.6 Pathname Expansion].""" - if self._env.has_opt('-f'): - return [wtree] - - # All expansions have been performed, only quoted sequences should remain - # in the tree. Generate the pattern by folding the tree, escaping special - # characters when appear quoted - special_chars = '*?[]' - - def make_pattern(wtree): - subpattern = [] - for part in wtree[1:-1]: - if isinstance(part, list): - part = make_pattern(part) - elif wtree[0]!='': - for c in part: - # Meta-characters cannot be quoted - if c in special_chars: - raise GlobError() - subpattern.append(part) - return ''.join(subpattern) - - def pwd_glob(pattern): - cwd = os.getcwd() - os.chdir(self._env['PWD']) - try: - return glob.glob(pattern) - finally: - os.chdir(cwd) - - #TODO: check working directory issues here wrt relative patterns - try: - pattern = make_pattern(wtree) - paths = pwd_glob(pattern) - except GlobError: - # BUG: Meta-characters were found in quoted sequences. The should - # have been used literally but this is unsupported in current glob module. - # Instead we consider the whole tree must be used literally and - # therefore there is no point in globbing. This is wrong when meta - # characters are mixed with quoted meta in the same pattern like: - # < foo*"py*" > - paths = [] - - if not paths: - return [wtree] - return [['', path, ''] for path in paths] - - def _remove_quotes(self, wtree): - """See [2.6.7 Quote Removal].""" - - def unquote(wtree): - unquoted = [] - for part in wtree[1:-1]: - if isinstance(part, list): - part = unquote(part) - unquoted.append(part) - return ''.join(unquoted) - - return ['', unquote(wtree), ''] - - def subshell(self, script=None, ast=None, redirs=None): - """Execute the script or AST in a subshell, with inherited redirections - if redirs is not None. - """ - if redirs: - sub_redirs = redirs - else: - sub_redirs = redirs.clone() - - subshell = None - try: - subshell = Interpreter(None, self._debugflags, self._env.clone(True), - sub_redirs, opts=self._options) - return subshell.execute_script(script, ast) - finally: - if not redirs: sub_redirs.close() - if subshell: subshell.close() - - def subshell_output(self, script): - """Execute the script in a subshell and return the captured output.""" - # Create temporary file to capture subshell output - tmpfd, tmppath = tempfile.mkstemp() - try: - tmpfile = os.fdopen(tmpfd, 'wb') - stdout = FileWrapper('w', tmpfile) - - redirs = Redirections(self._redirs.stdin().dup(), - stdout, - self._redirs.stderr().dup()) - try: - status = self.subshell(script=script, redirs=redirs) - finally: - redirs.close() - redirs = None - - # Extract subshell standard output - tmpfile = open(tmppath, 'rb') - try: - output = tmpfile.read() - return status, output.rstrip('\n') - finally: - tmpfile.close() - finally: - os.remove(tmppath) - - def _asynclist(self, cmd): - args = (self._env.get_variables(), cmd) - arg = encodeargs(args) - assert len(args) < 30*1024 - cmd = ['pysh.bat', '--ast', '-c', arg] - p = subprocess.Popen(cmd, cwd=self._env['PWD']) - self._children[p.pid] = p - self._env['!'] = p.pid - return 0 - - def wait(self, pids=None): - if not pids: - pids = self._children.keys() - - status = 127 - for pid in pids: - if pid not in self._children: - continue - p = self._children.pop(pid) - status = p.wait() - - return status - diff --git a/bitbake/lib/bb/pysh/lsprof.py b/bitbake/lib/bb/pysh/lsprof.py deleted file mode 100644 index b1831c22a7..0000000000 --- a/bitbake/lib/bb/pysh/lsprof.py +++ /dev/null @@ -1,116 +0,0 @@ -#! /usr/bin/env python - -import sys -from _lsprof import Profiler, profiler_entry - -__all__ = ['profile', 'Stats'] - -def profile(f, *args, **kwds): - """XXX docstring""" - p = Profiler() - p.enable(subcalls=True, builtins=True) - try: - f(*args, **kwds) - finally: - p.disable() - return Stats(p.getstats()) - - -class Stats(object): - """XXX docstring""" - - def __init__(self, data): - self.data = data - - def sort(self, crit="inlinetime"): - """XXX docstring""" - if crit not in profiler_entry.__dict__: - raise ValueError("Can't sort by %s" % crit) - self.data.sort(lambda b, a: cmp(getattr(a, crit), - getattr(b, crit))) - for e in self.data: - if e.calls: - e.calls.sort(lambda b, a: cmp(getattr(a, crit), - getattr(b, crit))) - - def pprint(self, top=None, file=None, limit=None, climit=None): - """XXX docstring""" - if file is None: - file = sys.stdout - d = self.data - if top is not None: - d = d[:top] - cols = "% 12s %12s %11.4f %11.4f %s\n" - hcols = "% 12s %12s %12s %12s %s\n" - cols2 = "+%12s %12s %11.4f %11.4f + %s\n" - file.write(hcols % ("CallCount", "Recursive", "Total(ms)", - "Inline(ms)", "module:lineno(function)")) - count = 0 - for e in d: - file.write(cols % (e.callcount, e.reccallcount, e.totaltime, - e.inlinetime, label(e.code))) - count += 1 - if limit is not None and count == limit: - return - ccount = 0 - if e.calls: - for se in e.calls: - file.write(cols % ("+%s" % se.callcount, se.reccallcount, - se.totaltime, se.inlinetime, - "+%s" % label(se.code))) - count += 1 - ccount += 1 - if limit is not None and count == limit: - return - if climit is not None and ccount == climit: - break - - def freeze(self): - """Replace all references to code objects with string - descriptions; this makes it possible to pickle the instance.""" - - # this code is probably rather ickier than it needs to be! - for i in range(len(self.data)): - e = self.data[i] - if not isinstance(e.code, str): - self.data[i] = type(e)((label(e.code),) + e[1:]) - if e.calls: - for j in range(len(e.calls)): - se = e.calls[j] - if not isinstance(se.code, str): - e.calls[j] = type(se)((label(se.code),) + se[1:]) - -_fn2mod = {} - -def label(code): - if isinstance(code, str): - return code - try: - mname = _fn2mod[code.co_filename] - except KeyError: - for k, v in sys.modules.items(): - if v is None: - continue - if not hasattr(v, '__file__'): - continue - if not isinstance(v.__file__, str): - continue - if v.__file__.startswith(code.co_filename): - mname = _fn2mod[code.co_filename] = k - break - else: - mname = _fn2mod[code.co_filename] = '<%s>'%code.co_filename - - return '%s:%d(%s)' % (mname, code.co_firstlineno, code.co_name) - - -if __name__ == '__main__': - import os - sys.argv = sys.argv[1:] - if not sys.argv: - print >> sys.stderr, "usage: lsprof.py <script> <arguments...>" - sys.exit(2) - sys.path.insert(0, os.path.abspath(os.path.dirname(sys.argv[0]))) - stats = profile(execfile, sys.argv[0], globals(), locals()) - stats.sort() - stats.pprint() diff --git a/bitbake/lib/bb/pysh/pysh.py b/bitbake/lib/bb/pysh/pysh.py deleted file mode 100644 index b4e6145b51..0000000000 --- a/bitbake/lib/bb/pysh/pysh.py +++ /dev/null @@ -1,167 +0,0 @@ -# pysh.py - command processing for pysh. -# -# Copyright 2007 Patrick Mezard -# -# This software may be used and distributed according to the terms -# of the GNU General Public License, incorporated herein by reference. - -import optparse -import os -import sys - -import interp - -SH_OPT = optparse.OptionParser(prog='pysh', usage="%prog [OPTIONS]", version='0.1') -SH_OPT.add_option('-c', action='store_true', dest='command_string', default=None, - help='A string that shall be interpreted by the shell as one or more commands') -SH_OPT.add_option('--redirect-to', dest='redirect_to', default=None, - help='Redirect script commands stdout and stderr to the specified file') -# See utility_command in builtin.py about the reason for this flag. -SH_OPT.add_option('--redirected', dest='redirected', action='store_true', default=False, - help='Tell the interpreter that stdout and stderr are actually the same objects, which is really stdout') -SH_OPT.add_option('--debug-parsing', action='store_true', dest='debug_parsing', default=False, - help='Trace PLY execution') -SH_OPT.add_option('--debug-tree', action='store_true', dest='debug_tree', default=False, - help='Display the generated syntax tree.') -SH_OPT.add_option('--debug-cmd', action='store_true', dest='debug_cmd', default=False, - help='Trace command execution before parameters expansion and exit status.') -SH_OPT.add_option('--debug-utility', action='store_true', dest='debug_utility', default=False, - help='Trace utility calls, after parameters expansions') -SH_OPT.add_option('--ast', action='store_true', dest='ast', default=False, - help='Encoded commands to execute in a subprocess') -SH_OPT.add_option('--profile', action='store_true', default=False, - help='Profile pysh run') - - -def split_args(args): - # Separate shell arguments from command ones - # Just stop at the first argument not starting with a dash. I know, this is completely broken, - # it ignores files starting with a dash or may take option values for command file. This is not - # supposed to happen for now - command_index = len(args) - for i,arg in enumerate(args): - if not arg.startswith('-'): - command_index = i - break - - return args[:command_index], args[command_index:] - - -def fixenv(env): - path = env.get('PATH') - if path is not None: - parts = path.split(os.pathsep) - # Remove Windows utilities from PATH, they are useless at best and - # some of them (find) may be confused with other utilities. - parts = [p for p in parts if 'system32' not in p.lower()] - env['PATH'] = os.pathsep.join(parts) - if env.get('HOME') is None: - # Several utilities, including cvsps, cannot work without - # a defined HOME directory. - env['HOME'] = os.path.expanduser('~') - return env - -def _sh(cwd, shargs, cmdargs, options, debugflags=None, env=None): - if os.environ.get('PYSH_TEXT') != '1': - import msvcrt - for fp in (sys.stdin, sys.stdout, sys.stderr): - msvcrt.setmode(fp.fileno(), os.O_BINARY) - - hgbin = os.environ.get('PYSH_HGTEXT') != '1' - - if debugflags is None: - debugflags = [] - if options.debug_parsing: debugflags.append('debug-parsing') - if options.debug_utility: debugflags.append('debug-utility') - if options.debug_cmd: debugflags.append('debug-cmd') - if options.debug_tree: debugflags.append('debug-tree') - - if env is None: - env = fixenv(dict(os.environ)) - if cwd is None: - cwd = os.getcwd() - - if not cmdargs: - # Nothing to do - return 0 - - ast = None - command_file = None - if options.command_string: - input = cmdargs[0] - if not options.ast: - input += '\n' - else: - args, input = interp.decodeargs(input), None - env, ast = args - cwd = env.get('PWD', cwd) - else: - command_file = cmdargs[0] - arguments = cmdargs[1:] - - prefix = interp.resolve_shebang(command_file, ignoreshell=True) - if prefix: - input = ' '.join(prefix + [command_file] + arguments) - else: - # Read commands from file - f = file(command_file) - try: - # Trailing newline to help the parser - input = f.read() + '\n' - finally: - f.close() - - redirect = None - try: - if options.redirected: - stdout = sys.stdout - stderr = stdout - elif options.redirect_to: - redirect = open(options.redirect_to, 'wb') - stdout = redirect - stderr = redirect - else: - stdout = sys.stdout - stderr = sys.stderr - - # TODO: set arguments to environment variables - opts = interp.Options() - opts.hgbinary = hgbin - ip = interp.Interpreter(cwd, debugflags, stdout=stdout, stderr=stderr, - opts=opts) - try: - # Export given environment in shell object - for k,v in env.iteritems(): - ip.get_env().export(k,v) - return ip.execute_script(input, ast, scriptpath=command_file) - finally: - ip.close() - finally: - if redirect is not None: - redirect.close() - -def sh(cwd=None, args=None, debugflags=None, env=None): - if args is None: - args = sys.argv[1:] - shargs, cmdargs = split_args(args) - options, shargs = SH_OPT.parse_args(shargs) - - if options.profile: - import lsprof - p = lsprof.Profiler() - p.enable(subcalls=True) - try: - return _sh(cwd, shargs, cmdargs, options, debugflags, env) - finally: - p.disable() - stats = lsprof.Stats(p.getstats()) - stats.sort() - stats.pprint(top=10, file=sys.stderr, climit=5) - else: - return _sh(cwd, shargs, cmdargs, options, debugflags, env) - -def main(): - sys.exit(sh()) - -if __name__=='__main__': - main() diff --git a/bitbake/lib/bb/pysh/pyshlex.py b/bitbake/lib/bb/pysh/pyshlex.py deleted file mode 100644 index b977b5e869..0000000000 --- a/bitbake/lib/bb/pysh/pyshlex.py +++ /dev/null @@ -1,888 +0,0 @@ -# pyshlex.py - PLY compatible lexer for pysh. -# -# Copyright 2007 Patrick Mezard -# -# This software may be used and distributed according to the terms -# of the GNU General Public License, incorporated herein by reference. - -# TODO: -# - review all "char in 'abc'" snippets: the empty string can be matched -# - test line continuations within quoted/expansion strings -# - eof is buggy wrt sublexers -# - the lexer cannot really work in pull mode as it would be required to run -# PLY in pull mode. It was designed to work incrementally and it would not be -# that hard to enable pull mode. -import re -try: - s = set() - del s -except NameError: - from Set import Set as set - -from ply import lex -from sherrors import * - -class NeedMore(Exception): - pass - -def is_blank(c): - return c in (' ', '\t') - -_RE_DIGITS = re.compile(r'^\d+$') - -def are_digits(s): - return _RE_DIGITS.search(s) is not None - -_OPERATORS = dict([ - ('&&', 'AND_IF'), - ('||', 'OR_IF'), - (';;', 'DSEMI'), - ('<<', 'DLESS'), - ('>>', 'DGREAT'), - ('<&', 'LESSAND'), - ('>&', 'GREATAND'), - ('<>', 'LESSGREAT'), - ('<<-', 'DLESSDASH'), - ('>|', 'CLOBBER'), - ('&', 'AMP'), - (';', 'COMMA'), - ('<', 'LESS'), - ('>', 'GREATER'), - ('(', 'LPARENS'), - (')', 'RPARENS'), -]) - -#Make a function to silence pychecker "Local variable shadows global" -def make_partial_ops(): - partials = {} - for k in _OPERATORS: - for i in range(1, len(k)+1): - partials[k[:i]] = None - return partials - -_PARTIAL_OPERATORS = make_partial_ops() - -def is_partial_op(s): - """Return True if s matches a non-empty subpart of an operator starting - at its first character. - """ - return s in _PARTIAL_OPERATORS - -def is_op(s): - """If s matches an operator, returns the operator identifier. Return None - otherwise. - """ - return _OPERATORS.get(s) - -_RESERVEDS = dict([ - ('if', 'If'), - ('then', 'Then'), - ('else', 'Else'), - ('elif', 'Elif'), - ('fi', 'Fi'), - ('do', 'Do'), - ('done', 'Done'), - ('case', 'Case'), - ('esac', 'Esac'), - ('while', 'While'), - ('until', 'Until'), - ('for', 'For'), - ('{', 'Lbrace'), - ('}', 'Rbrace'), - ('!', 'Bang'), - ('in', 'In'), - ('|', 'PIPE'), -]) - -def get_reserved(s): - return _RESERVEDS.get(s) - -_RE_NAME = re.compile(r'^[0-9a-zA-Z_]+$') - -def is_name(s): - return _RE_NAME.search(s) is not None - -def find_chars(seq, chars): - for i,v in enumerate(seq): - if v in chars: - return i,v - return -1, None - -class WordLexer: - """WordLexer parse quoted or expansion expressions and return an expression - tree. The input string can be any well formed sequence beginning with quoting - or expansion character. Embedded expressions are handled recursively. The - resulting tree is made of lists and strings. Lists represent quoted or - expansion expressions. Each list first element is the opening separator, - the last one the closing separator. In-between can be any number of strings - or lists for sub-expressions. Non quoted/expansion expression can written as - strings or as lists with empty strings as starting and ending delimiters. - """ - - NAME_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_' - NAME_CHARSET = dict(zip(NAME_CHARSET, NAME_CHARSET)) - - SPECIAL_CHARSET = '@*#?-$!0' - - #Characters which can be escaped depends on the current delimiters - ESCAPABLE = { - '`': set(['$', '\\', '`']), - '"': set(['$', '\\', '`', '"']), - "'": set(), - } - - def __init__(self, heredoc = False): - # _buffer is the unprocessed input characters buffer - self._buffer = [] - # _stack is empty or contains a quoted list being processed - # (this is the DFS path to the quoted expression being evaluated). - self._stack = [] - self._escapable = None - # True when parsing unquoted here documents - self._heredoc = heredoc - - def add(self, data, eof=False): - """Feed the lexer with more data. If the quoted expression can be - delimited, return a tuple (expr, remaining) containing the expression - tree and the unconsumed data. - Otherwise, raise NeedMore. - """ - self._buffer += list(data) - self._parse(eof) - - result = self._stack[0] - remaining = ''.join(self._buffer) - self._stack = [] - self._buffer = [] - return result, remaining - - def _is_escapable(self, c, delim=None): - if delim is None: - if self._heredoc: - # Backslashes works as if they were double quoted in unquoted - # here-documents - delim = '"' - else: - if len(self._stack)<=1: - return True - delim = self._stack[-2][0] - - escapables = self.ESCAPABLE.get(delim, None) - return escapables is None or c in escapables - - def _parse_squote(self, buf, result, eof): - if not buf: - raise NeedMore() - try: - pos = buf.index("'") - except ValueError: - raise NeedMore() - result[-1] += ''.join(buf[:pos]) - result += ["'"] - return pos+1, True - - def _parse_bquote(self, buf, result, eof): - if not buf: - raise NeedMore() - - if buf[0]=='\n': - #Remove line continuations - result[:] = ['', '', ''] - elif self._is_escapable(buf[0]): - result[-1] += buf[0] - result += [''] - else: - #Keep as such - result[:] = ['', '\\'+buf[0], ''] - - return 1, True - - def _parse_dquote(self, buf, result, eof): - if not buf: - raise NeedMore() - pos, sep = find_chars(buf, '$\\`"') - if pos==-1: - raise NeedMore() - - result[-1] += ''.join(buf[:pos]) - if sep=='"': - result += ['"'] - return pos+1, True - else: - #Keep everything until the separator and defer processing - return pos, False - - def _parse_command(self, buf, result, eof): - if not buf: - raise NeedMore() - - chars = '$\\`"\'' - if result[0] == '$(': - chars += ')' - pos, sep = find_chars(buf, chars) - if pos == -1: - raise NeedMore() - - result[-1] += ''.join(buf[:pos]) - if (result[0]=='$(' and sep==')') or (result[0]=='`' and sep=='`'): - result += [sep] - return pos+1, True - else: - return pos, False - - def _parse_parameter(self, buf, result, eof): - if not buf: - raise NeedMore() - - pos, sep = find_chars(buf, '$\\`"\'}') - if pos==-1: - raise NeedMore() - - result[-1] += ''.join(buf[:pos]) - if sep=='}': - result += [sep] - return pos+1, True - else: - return pos, False - - def _parse_dollar(self, buf, result, eof): - sep = result[0] - if sep=='$': - if not buf: - #TODO: handle empty $ - raise NeedMore() - if buf[0]=='(': - if len(buf)==1: - raise NeedMore() - - if buf[1]=='(': - result[0] = '$((' - buf[:2] = [] - else: - result[0] = '$(' - buf[:1] = [] - - elif buf[0]=='{': - result[0] = '${' - buf[:1] = [] - else: - if buf[0] in self.SPECIAL_CHARSET: - result[-1] = buf[0] - read = 1 - else: - for read,c in enumerate(buf): - if c not in self.NAME_CHARSET: - break - else: - if not eof: - raise NeedMore() - read += 1 - - result[-1] += ''.join(buf[0:read]) - - if not result[-1]: - result[:] = ['', result[0], ''] - else: - result += [''] - return read,True - - sep = result[0] - if sep=='$(': - parsefunc = self._parse_command - elif sep=='${': - parsefunc = self._parse_parameter - else: - raise NotImplementedError() - - pos, closed = parsefunc(buf, result, eof) - return pos, closed - - def _parse(self, eof): - buf = self._buffer - stack = self._stack - recurse = False - - while 1: - if not stack or recurse: - if not buf: - raise NeedMore() - if buf[0] not in ('"\\`$\''): - raise ShellSyntaxError('Invalid quoted string sequence') - stack.append([buf[0], '']) - buf[:1] = [] - recurse = False - - result = stack[-1] - if result[0]=="'": - parsefunc = self._parse_squote - elif result[0]=='\\': - parsefunc = self._parse_bquote - elif result[0]=='"': - parsefunc = self._parse_dquote - elif result[0]=='`': - parsefunc = self._parse_command - elif result[0][0]=='$': - parsefunc = self._parse_dollar - else: - raise NotImplementedError() - - read, closed = parsefunc(buf, result, eof) - - buf[:read] = [] - if closed: - if len(stack)>1: - #Merge in parent expression - parsed = stack.pop() - stack[-1] += [parsed] - stack[-1] += [''] - else: - break - else: - recurse = True - -def normalize_wordtree(wtree): - """Fold back every literal sequence (delimited with empty strings) into - parent sequence. - """ - def normalize(wtree): - result = [] - for part in wtree[1:-1]: - if isinstance(part, list): - part = normalize(part) - if part[0]=='': - #Move the part content back at current level - result += part[1:-1] - continue - elif not part: - #Remove empty strings - continue - result.append(part) - if not result: - result = [''] - return [wtree[0]] + result + [wtree[-1]] - - return normalize(wtree) - - -def make_wordtree(token, here_document=False): - """Parse a delimited token and return a tree similar to the ones returned by - WordLexer. token may contain any combinations of expansion/quoted fields and - non-ones. - """ - tree = [''] - remaining = token - delimiters = '\\$`' - if not here_document: - delimiters += '\'"' - - while 1: - pos, sep = find_chars(remaining, delimiters) - if pos==-1: - tree += [remaining, ''] - return normalize_wordtree(tree) - tree.append(remaining[:pos]) - remaining = remaining[pos:] - - try: - result, remaining = WordLexer(heredoc = here_document).add(remaining, True) - except NeedMore: - raise ShellSyntaxError('Invalid token "%s"') - tree.append(result) - - -def wordtree_as_string(wtree): - """Rewrite an expression tree generated by make_wordtree as string.""" - def visit(node, output): - for child in node: - if isinstance(child, list): - visit(child, output) - else: - output.append(child) - - output = [] - visit(wtree, output) - return ''.join(output) - - -def unquote_wordtree(wtree): - """Fold the word tree while removing quotes everywhere. Other expansion - sequences are joined as such. - """ - def unquote(wtree): - unquoted = [] - if wtree[0] in ('', "'", '"', '\\'): - wtree = wtree[1:-1] - - for part in wtree: - if isinstance(part, list): - part = unquote(part) - unquoted.append(part) - return ''.join(unquoted) - - return unquote(wtree) - - -class HereDocLexer: - """HereDocLexer delimits whatever comes from the here-document starting newline - not included to the closing delimiter line included. - """ - def __init__(self, op, delim): - assert op in ('<<', '<<-') - if not delim: - raise ShellSyntaxError('invalid here document delimiter %s' % str(delim)) - - self._op = op - self._delim = delim - self._buffer = [] - self._token = [] - - def add(self, data, eof): - """If the here-document was delimited, return a tuple (content, remaining). - Raise NeedMore() otherwise. - """ - self._buffer += list(data) - self._parse(eof) - token = ''.join(self._token) - remaining = ''.join(self._buffer) - self._token, self._remaining = [], [] - return token, remaining - - def _parse(self, eof): - while 1: - #Look for first unescaped newline. Quotes may be ignored - escaped = False - for i,c in enumerate(self._buffer): - if escaped: - escaped = False - elif c=='\\': - escaped = True - elif c=='\n': - break - else: - i = -1 - - if i==-1 or self._buffer[i]!='\n': - if not eof: - raise NeedMore() - #No more data, maybe the last line is closing delimiter - line = ''.join(self._buffer) - eol = '' - self._buffer[:] = [] - else: - line = ''.join(self._buffer[:i]) - eol = self._buffer[i] - self._buffer[:i+1] = [] - - if self._op=='<<-': - line = line.lstrip('\t') - - if line==self._delim: - break - - self._token += [line, eol] - if i==-1: - break - -class Token: - #TODO: check this is still in use - OPERATOR = 'OPERATOR' - WORD = 'WORD' - - def __init__(self): - self.value = '' - self.type = None - - def __getitem__(self, key): - #Behave like a two elements tuple - if key==0: - return self.type - if key==1: - return self.value - raise IndexError(key) - - -class HereDoc: - def __init__(self, op, name=None): - self.op = op - self.name = name - self.pendings = [] - -TK_COMMA = 'COMMA' -TK_AMPERSAND = 'AMP' -TK_OP = 'OP' -TK_TOKEN = 'TOKEN' -TK_COMMENT = 'COMMENT' -TK_NEWLINE = 'NEWLINE' -TK_IONUMBER = 'IO_NUMBER' -TK_ASSIGNMENT = 'ASSIGNMENT_WORD' -TK_HERENAME = 'HERENAME' - -class Lexer: - """Main lexer. - - Call add() until the script AST is returned. - """ - # Here-document handling makes the whole thing more complex because they basically - # force tokens to be reordered: here-content must come right after the operator - # and the here-document name, while some other tokens might be following the - # here-document expression on the same line. - # - # So, here-doc states are basically: - # *self._state==ST_NORMAL - # - self._heredoc.op is None: no here-document - # - self._heredoc.op is not None but name is: here-document operator matched, - # waiting for the document name/delimiter - # - self._heredoc.op and name are not None: here-document is ready, following - # tokens are being stored and will be pushed again when the document is - # completely parsed. - # *self._state==ST_HEREDOC - # - The here-document is being delimited by self._herelexer. Once it is done - # the content is pushed in front of the pending token list then all these - # tokens are pushed once again. - ST_NORMAL = 'ST_NORMAL' - ST_OP = 'ST_OP' - ST_BACKSLASH = 'ST_BACKSLASH' - ST_QUOTED = 'ST_QUOTED' - ST_COMMENT = 'ST_COMMENT' - ST_HEREDOC = 'ST_HEREDOC' - - #Match end of backquote strings - RE_BACKQUOTE_END = re.compile(r'(?<!\\)(`)') - - def __init__(self, parent_state = None): - self._input = [] - self._pos = 0 - - self._token = '' - self._type = TK_TOKEN - - self._state = self.ST_NORMAL - self._parent_state = parent_state - self._wordlexer = None - - self._heredoc = HereDoc(None) - self._herelexer = None - - ### Following attributes are not used for delimiting token and can safely - ### be changed after here-document detection (see _push_toke) - - # Count the number of tokens following a 'For' reserved word. Needed to - # return an 'In' reserved word if it comes in third place. - self._for_count = None - - def add(self, data, eof=False): - """Feed the lexer with data. - - When eof is set to True, returns unconsumed data or raise if the lexer - is in the middle of a delimiting operation. - Raise NeedMore otherwise. - """ - self._input += list(data) - self._parse(eof) - self._input[:self._pos] = [] - return ''.join(self._input) - - def _parse(self, eof): - while self._state: - if self._pos>=len(self._input): - if not eof: - raise NeedMore() - elif self._state not in (self.ST_OP, self.ST_QUOTED, self.ST_HEREDOC): - #Delimit the current token and leave cleanly - self._push_token('') - break - else: - #Let the sublexer handle the eof themselves - pass - - if self._state==self.ST_NORMAL: - self._parse_normal() - elif self._state==self.ST_COMMENT: - self._parse_comment() - elif self._state==self.ST_OP: - self._parse_op(eof) - elif self._state==self.ST_QUOTED: - self._parse_quoted(eof) - elif self._state==self.ST_HEREDOC: - self._parse_heredoc(eof) - else: - assert False, "Unknown state " + str(self._state) - - if self._heredoc.op is not None: - raise ShellSyntaxError('missing here-document delimiter') - - def _parse_normal(self): - c = self._input[self._pos] - if c=='\n': - self._push_token(c) - self._token = c - self._type = TK_NEWLINE - self._push_token('') - self._pos += 1 - elif c in ('\\', '\'', '"', '`', '$'): - self._state = self.ST_QUOTED - elif is_partial_op(c): - self._push_token(c) - - self._type = TK_OP - self._token += c - self._pos += 1 - self._state = self.ST_OP - elif is_blank(c): - self._push_token(c) - - #Discard blanks - self._pos += 1 - elif self._token: - self._token += c - self._pos += 1 - elif c=='#': - self._state = self.ST_COMMENT - self._type = TK_COMMENT - self._pos += 1 - else: - self._pos += 1 - self._token += c - - def _parse_op(self, eof): - assert self._token - - while 1: - if self._pos>=len(self._input): - if not eof: - raise NeedMore() - c = '' - else: - c = self._input[self._pos] - - op = self._token + c - if c and is_partial_op(op): - #Still parsing an operator - self._token = op - self._pos += 1 - else: - #End of operator - self._push_token(c) - self._state = self.ST_NORMAL - break - - def _parse_comment(self): - while 1: - if self._pos>=len(self._input): - raise NeedMore() - - c = self._input[self._pos] - if c=='\n': - #End of comment, do not consume the end of line - self._state = self.ST_NORMAL - break - else: - self._token += c - self._pos += 1 - - def _parse_quoted(self, eof): - """Precondition: the starting backquote/dollar is still in the input queue.""" - if not self._wordlexer: - self._wordlexer = WordLexer() - - if self._pos<len(self._input): - #Transfer input queue character into the subparser - input = self._input[self._pos:] - self._pos += len(input) - - wtree, remaining = self._wordlexer.add(input, eof) - self._wordlexer = None - self._token += wordtree_as_string(wtree) - - #Put unparsed character back in the input queue - if remaining: - self._input[self._pos:self._pos] = list(remaining) - self._state = self.ST_NORMAL - - def _parse_heredoc(self, eof): - assert not self._token - - if self._herelexer is None: - self._herelexer = HereDocLexer(self._heredoc.op, self._heredoc.name) - - if self._pos<len(self._input): - #Transfer input queue character into the subparser - input = self._input[self._pos:] - self._pos += len(input) - - self._token, remaining = self._herelexer.add(input, eof) - - #Reset here-document state - self._herelexer = None - heredoc, self._heredoc = self._heredoc, HereDoc(None) - if remaining: - self._input[self._pos:self._pos] = list(remaining) - self._state = self.ST_NORMAL - - #Push pending tokens - heredoc.pendings[:0] = [(self._token, self._type, heredoc.name)] - for token, type, delim in heredoc.pendings: - self._token = token - self._type = type - self._push_token(delim) - - def _push_token(self, delim): - if not self._token: - return 0 - - if self._heredoc.op is not None: - if self._heredoc.name is None: - #Here-document name - if self._type!=TK_TOKEN: - raise ShellSyntaxError("expecting here-document name, got '%s'" % self._token) - self._heredoc.name = unquote_wordtree(make_wordtree(self._token)) - self._type = TK_HERENAME - else: - #Capture all tokens until the newline starting the here-document - if self._type==TK_NEWLINE: - assert self._state==self.ST_NORMAL - self._state = self.ST_HEREDOC - - self._heredoc.pendings.append((self._token, self._type, delim)) - self._token = '' - self._type = TK_TOKEN - return 1 - - # BEWARE: do not change parser state from here to the end of the function: - # when parsing between an here-document operator to the end of the line - # tokens are stored in self._heredoc.pendings. Therefore, they will not - # reach the section below. - - #Check operators - if self._type==TK_OP: - #False positive because of partial op matching - op = is_op(self._token) - if not op: - self._type = TK_TOKEN - else: - #Map to the specific operator - self._type = op - if self._token in ('<<', '<<-'): - #Done here rather than in _parse_op because there is no need - #to change the parser state since we are still waiting for - #the here-document name - if self._heredoc.op is not None: - raise ShellSyntaxError("syntax error near token '%s'" % self._token) - assert self._heredoc.op is None - self._heredoc.op = self._token - - if self._type==TK_TOKEN: - if '=' in self._token and not delim: - if self._token.startswith('='): - #Token is a WORD... a TOKEN that is. - pass - else: - prev = self._token[:self._token.find('=')] - if is_name(prev): - self._type = TK_ASSIGNMENT - else: - #Just a token (unspecified) - pass - else: - reserved = get_reserved(self._token) - if reserved is not None: - if reserved=='In' and self._for_count!=2: - #Sorry, not a reserved word after all - pass - else: - self._type = reserved - if reserved in ('For', 'Case'): - self._for_count = 0 - elif are_digits(self._token) and delim in ('<', '>'): - #Detect IO_NUMBER - self._type = TK_IONUMBER - elif self._token==';': - self._type = TK_COMMA - elif self._token=='&': - self._type = TK_AMPERSAND - elif self._type==TK_COMMENT: - #Comments are not part of sh grammar, ignore them - self._token = '' - self._type = TK_TOKEN - return 0 - - if self._for_count is not None: - #Track token count in 'For' expression to detect 'In' reserved words. - #Can only be in third position, no need to go beyond - self._for_count += 1 - if self._for_count==3: - self._for_count = None - - self.on_token((self._token, self._type)) - self._token = '' - self._type = TK_TOKEN - return 1 - - def on_token(self, token): - raise NotImplementedError - - -tokens = [ - TK_TOKEN, -# To silence yacc unused token warnings -# TK_COMMENT, - TK_NEWLINE, - TK_IONUMBER, - TK_ASSIGNMENT, - TK_HERENAME, -] - -#Add specific operators -tokens += _OPERATORS.values() -#Add reserved words -tokens += _RESERVEDS.values() - -class PLYLexer(Lexer): - """Bridge Lexer and PLY lexer interface.""" - def __init__(self): - Lexer.__init__(self) - self._tokens = [] - self._current = 0 - self.lineno = 0 - - def on_token(self, token): - value, type = token - - self.lineno = 0 - t = lex.LexToken() - t.value = value - t.type = type - t.lexer = self - t.lexpos = 0 - t.lineno = 0 - - self._tokens.append(t) - - def is_empty(self): - return not bool(self._tokens) - - #PLY compliant interface - def token(self): - if self._current>=len(self._tokens): - return None - t = self._tokens[self._current] - self._current += 1 - return t - - -def get_tokens(s): - """Parse the input string and return a tuple (tokens, unprocessed) where - tokens is a list of parsed tokens and unprocessed is the part of the input - string left untouched by the lexer. - """ - lexer = PLYLexer() - untouched = lexer.add(s, True) - tokens = [] - while 1: - token = lexer.token() - if token is None: - break - tokens.append(token) - - tokens = [(t.value, t.type) for t in tokens] - return tokens, untouched diff --git a/bitbake/lib/bb/pysh/pyshyacc.py b/bitbake/lib/bb/pysh/pyshyacc.py deleted file mode 100644 index e8e80aac45..0000000000 --- a/bitbake/lib/bb/pysh/pyshyacc.py +++ /dev/null @@ -1,779 +0,0 @@ -# pyshyacc.py - PLY grammar definition for pysh -# -# Copyright 2007 Patrick Mezard -# -# This software may be used and distributed according to the terms -# of the GNU General Public License, incorporated herein by reference. - -"""PLY grammar file. -""" -import os.path -import sys - -import pyshlex -tokens = pyshlex.tokens - -from ply import yacc -import sherrors - -class IORedirect: - def __init__(self, op, filename, io_number=None): - self.op = op - self.filename = filename - self.io_number = io_number - -class HereDocument: - def __init__(self, op, name, content, io_number=None): - self.op = op - self.name = name - self.content = content - self.io_number = io_number - -def make_io_redirect(p): - """Make an IORedirect instance from the input 'io_redirect' production.""" - name, io_number, io_target = p - assert name=='io_redirect' - - if io_target[0]=='io_file': - io_type, io_op, io_file = io_target - return IORedirect(io_op, io_file, io_number) - elif io_target[0]=='io_here': - io_type, io_op, io_name, io_content = io_target - return HereDocument(io_op, io_name, io_content, io_number) - else: - assert False, "Invalid IO redirection token %s" % repr(io_type) - -class SimpleCommand: - """ - assigns contains (name, value) pairs. - """ - def __init__(self, words, redirs, assigns): - self.words = list(words) - self.redirs = list(redirs) - self.assigns = list(assigns) - -class Pipeline: - def __init__(self, commands, reverse_status=False): - self.commands = list(commands) - assert self.commands #Grammar forbids this - self.reverse_status = reverse_status - -class AndOr: - def __init__(self, op, left, right): - self.op = str(op) - self.left = left - self.right = right - -class ForLoop: - def __init__(self, name, items, cmds): - self.name = str(name) - self.items = list(items) - self.cmds = list(cmds) - -class WhileLoop: - def __init__(self, condition, cmds): - self.condition = list(condition) - self.cmds = list(cmds) - -class UntilLoop: - def __init__(self, condition, cmds): - self.condition = list(condition) - self.cmds = list(cmds) - -class FunDef: - def __init__(self, name, body): - self.name = str(name) - self.body = body - -class BraceGroup: - def __init__(self, cmds): - self.cmds = list(cmds) - -class IfCond: - def __init__(self, cond, if_cmds, else_cmds): - self.cond = list(cond) - self.if_cmds = if_cmds - self.else_cmds = else_cmds - -class Case: - def __init__(self, name, items): - self.name = name - self.items = items - -class SubShell: - def __init__(self, cmds): - self.cmds = cmds - -class RedirectList: - def __init__(self, cmd, redirs): - self.cmd = cmd - self.redirs = list(redirs) - -def get_production(productions, ptype): - """productions must be a list of production tuples like (name, obj) where - name is the production string identifier. - Return the first production named 'ptype'. Raise KeyError if None can be - found. - """ - for production in productions: - if production is not None and production[0]==ptype: - return production - raise KeyError(ptype) - -#------------------------------------------------------------------------------- -# PLY grammar definition -#------------------------------------------------------------------------------- - -def p_multiple_commands(p): - """multiple_commands : newline_sequence - | complete_command - | multiple_commands complete_command""" - if len(p)==2: - if p[1] is not None: - p[0] = [p[1]] - else: - p[0] = [] - else: - p[0] = p[1] + [p[2]] - -def p_complete_command(p): - """complete_command : list separator - | list""" - if len(p)==3 and p[2] and p[2][1] == '&': - p[0] = ('async', p[1]) - else: - p[0] = p[1] - -def p_list(p): - """list : list separator_op and_or - | and_or""" - if len(p)==2: - p[0] = [p[1]] - else: - #if p[2]!=';': - # raise NotImplementedError('AND-OR list asynchronous execution is not implemented') - p[0] = p[1] + [p[3]] - -def p_and_or(p): - """and_or : pipeline - | and_or AND_IF linebreak pipeline - | and_or OR_IF linebreak pipeline""" - if len(p)==2: - p[0] = p[1] - else: - p[0] = ('and_or', AndOr(p[2], p[1], p[4])) - -def p_maybe_bang_word(p): - """maybe_bang_word : Bang""" - p[0] = ('maybe_bang_word', p[1]) - -def p_pipeline(p): - """pipeline : pipe_sequence - | bang_word pipe_sequence""" - if len(p)==3: - p[0] = ('pipeline', Pipeline(p[2][1:], True)) - else: - p[0] = ('pipeline', Pipeline(p[1][1:])) - -def p_pipe_sequence(p): - """pipe_sequence : command - | pipe_sequence PIPE linebreak command""" - if len(p)==2: - p[0] = ['pipe_sequence', p[1]] - else: - p[0] = p[1] + [p[4]] - -def p_command(p): - """command : simple_command - | compound_command - | compound_command redirect_list - | function_definition""" - - if p[1][0] in ( 'simple_command', - 'for_clause', - 'while_clause', - 'until_clause', - 'case_clause', - 'if_clause', - 'function_definition', - 'subshell', - 'brace_group',): - if len(p) == 2: - p[0] = p[1] - else: - p[0] = ('redirect_list', RedirectList(p[1], p[2][1:])) - else: - raise NotImplementedError('%s command is not implemented' % repr(p[1][0])) - -def p_compound_command(p): - """compound_command : brace_group - | subshell - | for_clause - | case_clause - | if_clause - | while_clause - | until_clause""" - p[0] = p[1] - -def p_subshell(p): - """subshell : LPARENS compound_list RPARENS""" - p[0] = ('subshell', SubShell(p[2][1:])) - -def p_compound_list(p): - """compound_list : term - | newline_list term - | term separator - | newline_list term separator""" - productions = p[1:] - try: - sep = get_production(productions, 'separator') - if sep[1]!=';': - raise NotImplementedError() - except KeyError: - pass - term = get_production(productions, 'term') - p[0] = ['compound_list'] + term[1:] - -def p_term(p): - """term : term separator and_or - | and_or""" - if len(p)==2: - p[0] = ['term', p[1]] - else: - if p[2] is not None and p[2][1] == '&': - p[0] = ['term', ('async', p[1][1:])] + [p[3]] - else: - p[0] = p[1] + [p[3]] - -def p_maybe_for_word(p): - # Rearrange 'For' priority wrt TOKEN. See p_for_word - """maybe_for_word : For""" - p[0] = ('maybe_for_word', p[1]) - -def p_for_clause(p): - """for_clause : for_word name linebreak do_group - | for_word name linebreak in sequential_sep do_group - | for_word name linebreak in wordlist sequential_sep do_group""" - productions = p[1:] - do_group = get_production(productions, 'do_group') - try: - items = get_production(productions, 'in')[1:] - except KeyError: - raise NotImplementedError('"in" omission is not implemented') - - try: - items = get_production(productions, 'wordlist')[1:] - except KeyError: - items = [] - - name = p[2] - p[0] = ('for_clause', ForLoop(name, items, do_group[1:])) - -def p_name(p): - """name : token""" #Was NAME instead of token - p[0] = p[1] - -def p_in(p): - """in : In""" - p[0] = ('in', p[1]) - -def p_wordlist(p): - """wordlist : wordlist token - | token""" - if len(p)==2: - p[0] = ['wordlist', ('TOKEN', p[1])] - else: - p[0] = p[1] + [('TOKEN', p[2])] - -def p_case_clause(p): - """case_clause : Case token linebreak in linebreak case_list Esac - | Case token linebreak in linebreak case_list_ns Esac - | Case token linebreak in linebreak Esac""" - if len(p) < 8: - items = [] - else: - items = p[6][1:] - name = p[2] - p[0] = ('case_clause', Case(name, [c[1] for c in items])) - -def p_case_list_ns(p): - """case_list_ns : case_list case_item_ns - | case_item_ns""" - p_case_list(p) - -def p_case_list(p): - """case_list : case_list case_item - | case_item""" - if len(p)==2: - p[0] = ['case_list', p[1]] - else: - p[0] = p[1] + [p[2]] - -def p_case_item_ns(p): - """case_item_ns : pattern RPARENS linebreak - | pattern RPARENS compound_list linebreak - | LPARENS pattern RPARENS linebreak - | LPARENS pattern RPARENS compound_list linebreak""" - p_case_item(p) - -def p_case_item(p): - """case_item : pattern RPARENS linebreak DSEMI linebreak - | pattern RPARENS compound_list DSEMI linebreak - | LPARENS pattern RPARENS linebreak DSEMI linebreak - | LPARENS pattern RPARENS compound_list DSEMI linebreak""" - if len(p) < 7: - name = p[1][1:] - else: - name = p[2][1:] - - try: - cmds = get_production(p[1:], "compound_list")[1:] - except KeyError: - cmds = [] - - p[0] = ('case_item', (name, cmds)) - -def p_pattern(p): - """pattern : token - | pattern PIPE token""" - if len(p)==2: - p[0] = ['pattern', ('TOKEN', p[1])] - else: - p[0] = p[1] + [('TOKEN', p[2])] - -def p_maybe_if_word(p): - # Rearrange 'If' priority wrt TOKEN. See p_if_word - """maybe_if_word : If""" - p[0] = ('maybe_if_word', p[1]) - -def p_maybe_then_word(p): - # Rearrange 'Then' priority wrt TOKEN. See p_then_word - """maybe_then_word : Then""" - p[0] = ('maybe_then_word', p[1]) - -def p_if_clause(p): - """if_clause : if_word compound_list then_word compound_list else_part Fi - | if_word compound_list then_word compound_list Fi""" - else_part = [] - if len(p)==7: - else_part = p[5] - p[0] = ('if_clause', IfCond(p[2][1:], p[4][1:], else_part)) - -def p_else_part(p): - """else_part : Elif compound_list then_word compound_list else_part - | Elif compound_list then_word compound_list - | Else compound_list""" - if len(p)==3: - p[0] = p[2][1:] - else: - else_part = [] - if len(p)==6: - else_part = p[5] - p[0] = ('elif', IfCond(p[2][1:], p[4][1:], else_part)) - -def p_while_clause(p): - """while_clause : While compound_list do_group""" - p[0] = ('while_clause', WhileLoop(p[2][1:], p[3][1:])) - -def p_maybe_until_word(p): - # Rearrange 'Until' priority wrt TOKEN. See p_until_word - """maybe_until_word : Until""" - p[0] = ('maybe_until_word', p[1]) - -def p_until_clause(p): - """until_clause : until_word compound_list do_group""" - p[0] = ('until_clause', UntilLoop(p[2][1:], p[3][1:])) - -def p_function_definition(p): - """function_definition : fname LPARENS RPARENS linebreak function_body""" - p[0] = ('function_definition', FunDef(p[1], p[5])) - -def p_function_body(p): - """function_body : compound_command - | compound_command redirect_list""" - if len(p)!=2: - raise NotImplementedError('functions redirections lists are not implemented') - p[0] = p[1] - -def p_fname(p): - """fname : TOKEN""" #Was NAME instead of token - p[0] = p[1] - -def p_brace_group(p): - """brace_group : Lbrace compound_list Rbrace""" - p[0] = ('brace_group', BraceGroup(p[2][1:])) - -def p_maybe_done_word(p): - #See p_assignment_word for details. - """maybe_done_word : Done""" - p[0] = ('maybe_done_word', p[1]) - -def p_maybe_do_word(p): - """maybe_do_word : Do""" - p[0] = ('maybe_do_word', p[1]) - -def p_do_group(p): - """do_group : do_word compound_list done_word""" - #Do group contains a list of AndOr - p[0] = ['do_group'] + p[2][1:] - -def p_simple_command(p): - """simple_command : cmd_prefix cmd_word cmd_suffix - | cmd_prefix cmd_word - | cmd_prefix - | cmd_name cmd_suffix - | cmd_name""" - words, redirs, assigns = [], [], [] - for e in p[1:]: - name = e[0] - if name in ('cmd_prefix', 'cmd_suffix'): - for sube in e[1:]: - subname = sube[0] - if subname=='io_redirect': - redirs.append(make_io_redirect(sube)) - elif subname=='ASSIGNMENT_WORD': - assigns.append(sube) - else: - words.append(sube) - elif name in ('cmd_word', 'cmd_name'): - words.append(e) - - cmd = SimpleCommand(words, redirs, assigns) - p[0] = ('simple_command', cmd) - -def p_cmd_name(p): - """cmd_name : TOKEN""" - p[0] = ('cmd_name', p[1]) - -def p_cmd_word(p): - """cmd_word : token""" - p[0] = ('cmd_word', p[1]) - -def p_maybe_assignment_word(p): - #See p_assignment_word for details. - """maybe_assignment_word : ASSIGNMENT_WORD""" - p[0] = ('maybe_assignment_word', p[1]) - -def p_cmd_prefix(p): - """cmd_prefix : io_redirect - | cmd_prefix io_redirect - | assignment_word - | cmd_prefix assignment_word""" - try: - prefix = get_production(p[1:], 'cmd_prefix') - except KeyError: - prefix = ['cmd_prefix'] - - try: - value = get_production(p[1:], 'assignment_word')[1] - value = ('ASSIGNMENT_WORD', value.split('=', 1)) - except KeyError: - value = get_production(p[1:], 'io_redirect') - p[0] = prefix + [value] - -def p_cmd_suffix(p): - """cmd_suffix : io_redirect - | cmd_suffix io_redirect - | token - | cmd_suffix token - | maybe_for_word - | cmd_suffix maybe_for_word - | maybe_done_word - | cmd_suffix maybe_done_word - | maybe_do_word - | cmd_suffix maybe_do_word - | maybe_until_word - | cmd_suffix maybe_until_word - | maybe_assignment_word - | cmd_suffix maybe_assignment_word - | maybe_if_word - | cmd_suffix maybe_if_word - | maybe_then_word - | cmd_suffix maybe_then_word - | maybe_bang_word - | cmd_suffix maybe_bang_word""" - try: - suffix = get_production(p[1:], 'cmd_suffix') - token = p[2] - except KeyError: - suffix = ['cmd_suffix'] - token = p[1] - - if isinstance(token, tuple): - if token[0]=='io_redirect': - p[0] = suffix + [token] - else: - #Convert maybe_* to TOKEN if necessary - p[0] = suffix + [('TOKEN', token[1])] - else: - p[0] = suffix + [('TOKEN', token)] - -def p_redirect_list(p): - """redirect_list : io_redirect - | redirect_list io_redirect""" - if len(p) == 2: - p[0] = ['redirect_list', make_io_redirect(p[1])] - else: - p[0] = p[1] + [make_io_redirect(p[2])] - -def p_io_redirect(p): - """io_redirect : io_file - | IO_NUMBER io_file - | io_here - | IO_NUMBER io_here""" - if len(p)==3: - p[0] = ('io_redirect', p[1], p[2]) - else: - p[0] = ('io_redirect', None, p[1]) - -def p_io_file(p): - #Return the tuple (operator, filename) - """io_file : LESS filename - | LESSAND filename - | GREATER filename - | GREATAND filename - | DGREAT filename - | LESSGREAT filename - | CLOBBER filename""" - #Extract the filename from the file - p[0] = ('io_file', p[1], p[2][1]) - -def p_filename(p): - #Return the filename - """filename : TOKEN""" - p[0] = ('filename', p[1]) - -def p_io_here(p): - """io_here : DLESS here_end - | DLESSDASH here_end""" - p[0] = ('io_here', p[1], p[2][1], p[2][2]) - -def p_here_end(p): - """here_end : HERENAME TOKEN""" - p[0] = ('here_document', p[1], p[2]) - -def p_newline_sequence(p): - # Nothing in the grammar can handle leading NEWLINE productions, so add - # this one with the lowest possible priority relatively to newline_list. - """newline_sequence : newline_list""" - p[0] = None - -def p_newline_list(p): - """newline_list : NEWLINE - | newline_list NEWLINE""" - p[0] = None - -def p_linebreak(p): - """linebreak : newline_list - | empty""" - p[0] = None - -def p_separator_op(p): - """separator_op : COMMA - | AMP""" - p[0] = p[1] - -def p_separator(p): - """separator : separator_op linebreak - | newline_list""" - if len(p)==2: - #Ignore newlines - p[0] = None - else: - #Keep the separator operator - p[0] = ('separator', p[1]) - -def p_sequential_sep(p): - """sequential_sep : COMMA linebreak - | newline_list""" - p[0] = None - -# Low priority TOKEN => for_word conversion. -# Let maybe_for_word be used as a token when necessary in higher priority -# rules. -def p_for_word(p): - """for_word : maybe_for_word""" - p[0] = p[1] - -def p_if_word(p): - """if_word : maybe_if_word""" - p[0] = p[1] - -def p_then_word(p): - """then_word : maybe_then_word""" - p[0] = p[1] - -def p_done_word(p): - """done_word : maybe_done_word""" - p[0] = p[1] - -def p_do_word(p): - """do_word : maybe_do_word""" - p[0] = p[1] - -def p_until_word(p): - """until_word : maybe_until_word""" - p[0] = p[1] - -def p_assignment_word(p): - """assignment_word : maybe_assignment_word""" - p[0] = ('assignment_word', p[1][1]) - -def p_bang_word(p): - """bang_word : maybe_bang_word""" - p[0] = ('bang_word', p[1][1]) - -def p_token(p): - """token : TOKEN - | Fi""" - p[0] = p[1] - -def p_empty(p): - 'empty :' - p[0] = None - -# Error rule for syntax errors -def p_error(p): - msg = [] - w = msg.append - w('%r\n' % p) - w('followed by:\n') - for i in range(5): - n = yacc.token() - if not n: - break - w(' %r\n' % n) - raise sherrors.ShellSyntaxError(''.join(msg)) - -# Build the parser -try: - import pyshtables -except ImportError: - outputdir = os.path.dirname(__file__) - if not os.access(outputdir, os.W_OK): - outputdir = '' - yacc.yacc(tabmodule = 'pyshtables', outputdir = outputdir, debug = 0) -else: - yacc.yacc(tabmodule = 'pysh.pyshtables', write_tables = 0, debug = 0) - - -def parse(input, eof=False, debug=False): - """Parse a whole script at once and return the generated AST and unconsumed - data in a tuple. - - NOTE: eof is probably meaningless for now, the parser being unable to work - in pull mode. It should be set to True. - """ - lexer = pyshlex.PLYLexer() - remaining = lexer.add(input, eof) - if lexer.is_empty(): - return [], remaining - if debug: - debug = 2 - return yacc.parse(lexer=lexer, debug=debug), remaining - -#------------------------------------------------------------------------------- -# AST rendering helpers -#------------------------------------------------------------------------------- - -def format_commands(v): - """Return a tree made of strings and lists. Make command trees easier to - display. - """ - if isinstance(v, list): - return [format_commands(c) for c in v] - if isinstance(v, tuple): - if len(v)==2 and isinstance(v[0], str) and not isinstance(v[1], str): - if v[0] == 'async': - return ['AsyncList', map(format_commands, v[1])] - else: - #Avoid decomposing tuples like ('pipeline', Pipeline(...)) - return format_commands(v[1]) - return format_commands(list(v)) - elif isinstance(v, IfCond): - name = ['IfCond'] - name += ['if', map(format_commands, v.cond)] - name += ['then', map(format_commands, v.if_cmds)] - name += ['else', map(format_commands, v.else_cmds)] - return name - elif isinstance(v, ForLoop): - name = ['ForLoop'] - name += [repr(v.name)+' in ', map(str, v.items)] - name += ['commands', map(format_commands, v.cmds)] - return name - elif isinstance(v, AndOr): - return [v.op, format_commands(v.left), format_commands(v.right)] - elif isinstance(v, Pipeline): - name = 'Pipeline' - if v.reverse_status: - name = '!' + name - return [name, format_commands(v.commands)] - elif isinstance(v, Case): - name = ['Case'] - name += [v.name, format_commands(v.items)] - elif isinstance(v, SimpleCommand): - name = ['SimpleCommand'] - if v.words: - name += ['words', map(str, v.words)] - if v.assigns: - assigns = [tuple(a[1]) for a in v.assigns] - name += ['assigns', map(str, assigns)] - if v.redirs: - name += ['redirs', map(format_commands, v.redirs)] - return name - elif isinstance(v, RedirectList): - name = ['RedirectList'] - if v.redirs: - name += ['redirs', map(format_commands, v.redirs)] - name += ['command', format_commands(v.cmd)] - return name - elif isinstance(v, IORedirect): - return ' '.join(map(str, (v.io_number, v.op, v.filename))) - elif isinstance(v, HereDocument): - return ' '.join(map(str, (v.io_number, v.op, repr(v.name), repr(v.content)))) - elif isinstance(v, SubShell): - return ['SubShell', map(format_commands, v.cmds)] - else: - return repr(v) - -def print_commands(cmds, output=sys.stdout): - """Pretty print a command tree.""" - def print_tree(cmd, spaces, output): - if isinstance(cmd, list): - for c in cmd: - print_tree(c, spaces + 3, output) - else: - print >>output, ' '*spaces + str(cmd) - - formatted = format_commands(cmds) - print_tree(formatted, 0, output) - - -def stringify_commands(cmds): - """Serialize a command tree as a string. - - Returned string is not pretty and is currently used for unit tests only. - """ - def stringify(value): - output = [] - if isinstance(value, list): - formatted = [] - for v in value: - formatted.append(stringify(v)) - formatted = ' '.join(formatted) - output.append(''.join(['<', formatted, '>'])) - else: - output.append(value) - return ' '.join(output) - - return stringify(format_commands(cmds)) - - -def visit_commands(cmds, callable): - """Visit the command tree and execute callable on every Pipeline and - SimpleCommand instances. - """ - if isinstance(cmds, (tuple, list)): - map(lambda c: visit_commands(c,callable), cmds) - elif isinstance(cmds, (Pipeline, SimpleCommand)): - callable(cmds) diff --git a/bitbake/lib/bb/pysh/sherrors.py b/bitbake/lib/bb/pysh/sherrors.py deleted file mode 100644 index 1d5bd53b3a..0000000000 --- a/bitbake/lib/bb/pysh/sherrors.py +++ /dev/null @@ -1,41 +0,0 @@ -# sherrors.py - shell errors and signals -# -# Copyright 2007 Patrick Mezard -# -# This software may be used and distributed according to the terms -# of the GNU General Public License, incorporated herein by reference. - -"""Define shell exceptions and error codes. -""" - -class ShellError(Exception): - pass - -class ShellSyntaxError(ShellError): - pass - -class UtilityError(ShellError): - """Raised upon utility syntax error (option or operand error).""" - pass - -class ExpansionError(ShellError): - pass - -class CommandNotFound(ShellError): - """Specified command was not found.""" - pass - -class RedirectionError(ShellError): - pass - -class VarAssignmentError(ShellError): - """Variable assignment error.""" - pass - -class ExitSignal(ShellError): - """Exit signal.""" - pass - -class ReturnSignal(ShellError): - """Exit signal.""" - pass
\ No newline at end of file diff --git a/bitbake/lib/bb/pysh/subprocess_fix.py b/bitbake/lib/bb/pysh/subprocess_fix.py deleted file mode 100644 index 46eca22802..0000000000 --- a/bitbake/lib/bb/pysh/subprocess_fix.py +++ /dev/null @@ -1,77 +0,0 @@ -# subprocess - Subprocesses with accessible I/O streams -# -# For more information about this module, see PEP 324. -# -# This module should remain compatible with Python 2.2, see PEP 291. -# -# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> -# -# Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/2.4/license for licensing details. - -def list2cmdline(seq): - """ - Translate a sequence of arguments into a command line - string, using the same rules as the MS C runtime: - - 1) Arguments are delimited by white space, which is either a - space or a tab. - - 2) A string surrounded by double quotation marks is - interpreted as a single argument, regardless of white space - contained within. A quoted string can be embedded in an - argument. - - 3) A double quotation mark preceded by a backslash is - interpreted as a literal double quotation mark. - - 4) Backslashes are interpreted literally, unless they - immediately precede a double quotation mark. - - 5) If backslashes immediately precede a double quotation mark, - every pair of backslashes is interpreted as a literal - backslash. If the number of backslashes is odd, the last - backslash escapes the next double quotation mark as - described in rule 3. - """ - - # See - # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp - result = [] - needquote = False - for arg in seq: - bs_buf = [] - - # Add a space to separate this argument from the others - if result: - result.append(' ') - - needquote = (" " in arg) or ("\t" in arg) or ("|" in arg) or arg == "" - if needquote: - result.append('"') - - for c in arg: - if c == '\\': - # Don't know if we need to double yet. - bs_buf.append(c) - elif c == '"': - # Double backspaces. - result.append('\\' * len(bs_buf)*2) - bs_buf = [] - result.append('\\"') - else: - # Normal char - if bs_buf: - result.extend(bs_buf) - bs_buf = [] - result.append(c) - - # Add remaining backspaces, if any. - if bs_buf: - result.extend(bs_buf) - - if needquote: - result.extend(bs_buf) - result.append('"') - - return ''.join(result) |