From 2d3a6d22e155911e39e4b7e323317f4a7cb1cb95 Mon Sep 17 00:00:00 2001 From: Daniel Istrate Date: Wed, 2 Dec 2015 16:40:53 +0200 Subject: scripts: oe-selftest Added new features. [YOCTO #8750] Allow oe-selftest to run custom test suites based on different criteria 1. Can run custom lists of tests based on different criteria: --run-tests-by eg: --run-tests-by module imagefeatures signing recipetool --run-tests-by id 1377 1273 935 --run-tests-by tag wic sstate bitbake 2. Can list tests based on different criteria: --list-tests-by eg: --list-tests-by module imagefeatures signing recipetool --list-tests-by id 1377 1273 935 --list-tests-by tag wic sstate bitbake 3. Can list all tags that have been set to test cases: --list-tags The list of tags should be kept as minimal as possible. This helps preview the tags used so far. To take advantage of the 'tag' feature: - add @tag(feature=<>) to testcases eg: @tag(feature='signing') for a single tag @tag(feature=(('signing', 'sstate')) or @tag(feature=['signing', 'sstate']) for multiple tags Signed-off-by: Daniel Istrate Signed-off-by: Ross Burton --- scripts/oe-selftest | 218 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 216 insertions(+), 2 deletions(-) diff --git a/scripts/oe-selftest b/scripts/oe-selftest index 9679962ec0..bc50b2a435 100755 --- a/scripts/oe-selftest +++ b/scripts/oe-selftest @@ -72,6 +72,12 @@ def get_args_parser(): group.add_argument('--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.') group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.') parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing") + group.add_argument('--run-tests-by', required=False, dest='run_tests_by', default=False, nargs='*', + help='run-tests-by ') + group.add_argument('--list-tests-by', required=False, dest='list_tests_by', default=False, nargs='*', + help='list-tests-by ') + group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true", + help='List all tags that have been set to test cases.') return parser @@ -156,6 +162,187 @@ def get_tests(exclusive_modules=[], include_hidden=False): return testslist + +class Tc: + def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None): + self.tcname = tcname + self.tcclass = tcclass + self.tcmodule = tcmodule + self.tcid = tcid + # A test case can have multiple tags (as list or as tuples) otherwise str suffice + self.tctag = tctag + self.fullpath = '.'.join(['oeqa', 'selftest', tcmodule, tcclass, tcname]) + + +def get_tests_from_module(tmod): + tlist = [] + prefix = 'oeqa.selftest.' + + try: + import importlib + modlib = importlib.import_module(tmod) + for mod in vars(modlib).values(): + if isinstance(mod, type(oeSelfTest)) and issubclass(mod, oeSelfTest) and mod is not oeSelfTest: + for test in dir(mod): + if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'): + # Get test case id and feature tag + # NOTE: if testcase decorator or feature tag not set will throw error + try: + tid = vars(mod)[test].test_case + except: + print 'DEBUG: tc id missing for ' + str(test) + tid = None + try: + ttag = vars(mod)[test].tag__feature + except: + # print 'DEBUG: feature tag missing for ' + str(test) + ttag = None + + # NOTE: for some reason lstrip() doesn't work for mod.__module__ + tlist.append(Tc(test, mod.__name__, mod.__module__.replace(prefix, ''), tid, ttag)) + except: + pass + + return tlist + + +def get_all_tests(): + tmodules = set() + testlist = [] + prefix = 'oeqa.selftest.' + + # Get all the test modules (except the hidden ones) + for tpath in oeqa.selftest.__path__: + files = sorted([f for f in os.listdir(tpath) if f.endswith('.py') and not + f.startswith(('_', '__')) and f != 'base.py']) + for f in files: + tmodules.add(prefix + f.rstrip('.py')) + + # Get all the tests from modules + tmodules = sorted(list(tmodules)) + + for tmod in tmodules: + testlist += get_tests_from_module(tmod) + + return testlist + + +def create_testsuite_by(criteria, keyword): + # Create a testsuite based on 'keyword' + # criteria: name, class, module, id, tag + # keyword: a list of tests, classes, modules, ids, tags + # NOTE: globing would be nice? + + ts = set() + all_tests = get_all_tests() + + if criteria == 'name': + for tc in all_tests: + if tc.tcname in keyword: + ts.add(tc.fullpath) + + elif criteria == 'class': + for tc in all_tests: + if tc.tcclass in keyword: + ts.add(tc.fullpath) + + elif criteria == 'module': + for tc in all_tests: + if tc.tcmodule in keyword: + ts.add(tc.fullpath) + elif criteria == 'id': + for tc in all_tests: + if str(tc.tcid) in keyword: + ts.add(tc.fullpath) + elif criteria == 'tag': + for tc in all_tests: + # tc can have multiple tags (as list or tuple) otherwise as str + if isinstance(tc.tctag, (list, tuple)): + for tag in tc.tctag: + if str(tag) in keyword: + ts.add(tc.fullpath) + elif tc.tctag in keyword: + ts.add(tc.fullpath) + + return sorted(list(ts)) + + +def get_testsuite_by(criteria, keyword): + # Get a testsuite based on 'keyword' + # criteria: name, class, module, id, tag + # keyword: a list of tests, classes, modules, ids, tags + # NOTE: globing would be nice? + ts = set() + all_tests = get_all_tests() + + if criteria == 'name': + for tc in all_tests: + if tc.tcname in keyword: + ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule)) + + elif criteria == 'class': + for tc in all_tests: + if tc.tcclass in keyword: + ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule)) + + elif criteria == 'module': + for tc in all_tests: + if tc.tcmodule in keyword: + ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule)) + elif criteria == 'id': + for tc in all_tests: + if str(tc.tcid) in keyword: + ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule)) + elif criteria == 'tag': + for tc in all_tests: + # tc can have multiple tags (as list or tuple) otherwise as str + if isinstance(tc.tctag, (list, tuple)): + for tag in tc.tctag: + if str(tag) in keyword: + ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule)) + elif str(tc.tctag) in keyword: + ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule)) + + return sorted(list(ts)) + + +def list_testsuite_by(criteria, keyword): + # Get a testsuite based on 'keyword' + # criteria: name, class, module, id, tag + # keyword: a list of tests, classes, modules, ids, tags + # NOTE: globing would be nice? + + ts = get_testsuite_by(criteria, keyword) + + print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module') + print '_' * 150 + for t in ts: + if isinstance(t[1], (tuple, list)): + print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4]) + else: + print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t + print '_' * 150 + print 'Filtering by:\t %s' % criteria + print 'Looking for:\t %s' % ', '.join(str(x) for x in keyword) + print 'Total found:\t %s' % len(ts) + + +def list_tags(): + # Get all tags set to test cases + # This is useful when setting tags to test cases + # The list of tags should be kept as minimal as possible + tags = set() + all_tests = get_all_tests() + + for tc in all_tests: + if isinstance(tc.tctag, (tuple, list)): + tags.update(set(tc.tctag)) + else: + tags.add(tc.tctag) + + print 'Tags:\t%s' % ', '.join(str(x) for x in tags) + + def main(): parser = get_args_parser() args = parser.parse_args() @@ -167,6 +354,29 @@ def main(): sys.path.extend(layer_libdirs) reload(oeqa.selftest) + if args.run_tests_by and len(args.run_tests_by) >= 2: + valid_options = ['name', 'class', 'module', 'id', 'tag'] + if args.run_tests_by[0] not in valid_options: + print '--run-tests-by %s not a valid option. Choose one of .' % args.run_tests_by[0] + return 1 + else: + criteria = args.run_tests_by[0] + keyword = args.run_tests_by[1:] + ts = create_testsuite_by(criteria, keyword) + + if args.list_tests_by and len(args.list_tests_by) >= 2: + valid_options = ['name', 'class', 'module', 'id', 'tag'] + if args.list_tests_by[0] not in valid_options: + print '--list-tests-by %s not a valid option. Choose one of .' % args.list_tests_by[0] + return 1 + else: + criteria = args.list_tests_by[0] + keyword = args.list_tests_by[1:] + list_testsuite_by(criteria, keyword) + + if args.list_tags: + list_tags() + if args.list_allclasses: args.list_modules = True @@ -195,7 +405,7 @@ def main(): print e pass - if args.run_tests or args.run_all_tests: + if args.run_tests or args.run_all_tests or args.run_tests_by: if not preflight_check(): return 1 @@ -235,7 +445,11 @@ def main(): coverage_process_start = os.environ["COVERAGE_PROCESS_START"] = coveragerc - testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False) + if args.run_tests_by: + testslist = ts + else: + testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False) + suite = unittest.TestSuite() loader = unittest.TestLoader() loader.sortTestMethodsUsing = None -- cgit v1.2.3