scripts: oe-selftest Added new features.

[YOCTO #8750] Allow oe-selftest to run custom test suites based on different criteria

1. Can run custom lists of tests based on different criteria:
   --run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>
   eg: --run-tests-by module imagefeatures signing recipetool
       --run-tests-by id 1377 1273 935
       --run-tests-by tag wic sstate bitbake
2. Can list tests based on different criteria:
   --list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>
   eg: --list-tests-by module imagefeatures signing recipetool
       --list-tests-by id 1377 1273 935
       --list-tests-by tag wic sstate bitbake
3. Can list all tags that have been set to test cases:
   --list-tags
   The list of tags should be kept as minimal as possible.
   This helps preview the tags used so far.

   To take advantage of the 'tag' feature:
   - add @tag(feature=<>) to testcases
   eg: @tag(feature='signing') for a single tag
       @tag(feature=(('signing', 'sstate')) or
       @tag(feature=['signing', 'sstate']) for multiple tags

Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>

1 files changed, 216 insertions, 2 deletions

diff --git a/scripts/oe-selftest b/scripts/oe-selftest
index 9679962ec0..bc50b2a435 100755
--- a/scripts/oe-selftest
+++ b/scripts/oe-selftest
@@ -72,6 +72,12 @@ def get_args_parser():
     group.add_argument('--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.')
     group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.')
     parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing")
+    group.add_argument('--run-tests-by', required=False, dest='run_tests_by', default=False, nargs='*',
+                       help='run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
+    group.add_argument('--list-tests-by', required=False, dest='list_tests_by', default=False, nargs='*',
+                       help='list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
+    group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true",
+                       help='List all tags that have been set to test cases.')
     return parser
 
 
@@ -156,6 +162,187 @@ def get_tests(exclusive_modules=[], include_hidden=False):
 
     return testslist
 
+
+class Tc:
+    def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None):
+        self.tcname = tcname
+        self.tcclass = tcclass
+        self.tcmodule = tcmodule
+        self.tcid = tcid
+        # A test case can have multiple tags (as list or as tuples) otherwise str suffice
+        self.tctag = tctag
+        self.fullpath = '.'.join(['oeqa', 'selftest', tcmodule, tcclass, tcname])
+
+
+def get_tests_from_module(tmod):
+    tlist = []
+    prefix = 'oeqa.selftest.'
+
+    try:
+        import importlib
+        modlib = importlib.import_module(tmod)
+        for mod in vars(modlib).values():
+            if isinstance(mod, type(oeSelfTest)) and issubclass(mod, oeSelfTest) and mod is not oeSelfTest:
+                for test in dir(mod):
+                    if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'):
+                        # Get test case id and feature tag
+                        # NOTE: if testcase decorator or feature tag not set will throw error
+                        try:
+                            tid = vars(mod)[test].test_case
+                        except:
+                            print 'DEBUG: tc id missing for ' + str(test)
+                            tid = None
+                        try:
+                            ttag = vars(mod)[test].tag__feature
+                        except:
+                            # print 'DEBUG: feature tag missing for ' + str(test)
+                            ttag = None
+
+                        # NOTE: for some reason lstrip() doesn't work for mod.__module__
+                        tlist.append(Tc(test, mod.__name__, mod.__module__.replace(prefix, ''), tid, ttag))
+    except:
+        pass
+
+    return tlist
+
+
+def get_all_tests():
+    tmodules = set()
+    testlist = []
+    prefix = 'oeqa.selftest.'
+
+    # Get all the test modules (except the hidden ones)
+    for tpath in oeqa.selftest.__path__:
+        files = sorted([f for f in os.listdir(tpath) if f.endswith('.py') and not
+                        f.startswith(('_', '__')) and f != 'base.py'])
+        for f in files:
+            tmodules.add(prefix + f.rstrip('.py'))
+
+    # Get all the tests from modules
+    tmodules = sorted(list(tmodules))
+
+    for tmod in tmodules:
+        testlist += get_tests_from_module(tmod)
+
+    return testlist
+
+
+def create_testsuite_by(criteria, keyword):
+    # Create a testsuite based on 'keyword'
+    # criteria: name, class, module, id, tag
+    # keyword: a list of tests, classes, modules, ids, tags
+    # NOTE: globing would be nice?
+
+    ts = set()
+    all_tests = get_all_tests()
+
+    if criteria == 'name':
+        for tc in all_tests:
+            if tc.tcname in keyword:
+                ts.add(tc.fullpath)
+
+    elif criteria == 'class':
+        for tc in all_tests:
+            if tc.tcclass in keyword:
+                ts.add(tc.fullpath)
+
+    elif criteria == 'module':
+        for tc in all_tests:
+            if tc.tcmodule in keyword:
+                ts.add(tc.fullpath)
+    elif criteria == 'id':
+        for tc in all_tests:
+            if str(tc.tcid) in keyword:
+                ts.add(tc.fullpath)
+    elif criteria == 'tag':
+        for tc in all_tests:
+            # tc can have multiple tags (as list or tuple) otherwise as str
+            if isinstance(tc.tctag, (list, tuple)):
+                for tag in tc.tctag:
+                    if str(tag) in keyword:
+                        ts.add(tc.fullpath)
+            elif tc.tctag in keyword:
+                ts.add(tc.fullpath)
+
+    return sorted(list(ts))
+
+
+def get_testsuite_by(criteria, keyword):
+    # Get a testsuite based on 'keyword'
+    # criteria: name, class, module, id, tag
+    # keyword: a list of tests, classes, modules, ids, tags
+    # NOTE: globing would be nice?
+    ts = set()
+    all_tests = get_all_tests()
+
+    if criteria == 'name':
+        for tc in all_tests:
+            if tc.tcname in keyword:
+                ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+
+    elif criteria == 'class':
+        for tc in all_tests:
+            if tc.tcclass in keyword:
+                ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+
+    elif criteria == 'module':
+        for tc in all_tests:
+            if tc.tcmodule in keyword:
+                ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+    elif criteria == 'id':
+        for tc in all_tests:
+            if str(tc.tcid) in keyword:
+                ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+    elif criteria == 'tag':
+        for tc in all_tests:
+            # tc can have multiple tags (as list or tuple) otherwise as str
+            if isinstance(tc.tctag, (list, tuple)):
+                for tag in tc.tctag:
+                    if str(tag) in keyword:
+                        ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+            elif str(tc.tctag) in keyword:
+                ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+
+    return sorted(list(ts))
+
+
+def list_testsuite_by(criteria, keyword):
+    # Get a testsuite based on 'keyword'
+    # criteria: name, class, module, id, tag
+    # keyword: a list of tests, classes, modules, ids, tags
+    # NOTE: globing would be nice?
+
+    ts = get_testsuite_by(criteria, keyword)
+
+    print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module')
+    print '_' * 150
+    for t in ts:
+        if isinstance(t[1], (tuple, list)):
+            print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4])
+        else:
+            print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t
+    print '_' * 150
+    print 'Filtering by:\t %s' % criteria
+    print 'Looking for:\t %s' % ', '.join(str(x) for x in keyword)
+    print 'Total found:\t %s' % len(ts)
+
+
+def list_tags():
+    # Get all tags set to test cases
+    # This is useful when setting tags to test cases
+    # The list of tags should be kept as minimal as possible
+    tags = set()
+    all_tests = get_all_tests()
+
+    for tc in all_tests:
+        if isinstance(tc.tctag, (tuple, list)):
+            tags.update(set(tc.tctag))
+        else:
+            tags.add(tc.tctag)
+
+    print 'Tags:\t%s' % ', '.join(str(x) for x in tags)
+
+
 def main():
     parser = get_args_parser()
     args = parser.parse_args()
@@ -167,6 +354,29 @@ def main():
     sys.path.extend(layer_libdirs)
     reload(oeqa.selftest)
 
+    if args.run_tests_by and len(args.run_tests_by) >= 2:
+        valid_options = ['name', 'class', 'module', 'id', 'tag']
+        if args.run_tests_by[0] not in valid_options:
+            print '--run-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.run_tests_by[0]
+            return 1
+        else:
+            criteria = args.run_tests_by[0]
+            keyword = args.run_tests_by[1:]
+            ts = create_testsuite_by(criteria, keyword)
+
+    if args.list_tests_by and len(args.list_tests_by) >= 2:
+        valid_options = ['name', 'class', 'module', 'id', 'tag']
+        if args.list_tests_by[0] not in valid_options:
+            print '--list-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.list_tests_by[0]
+            return 1
+        else:
+            criteria = args.list_tests_by[0]
+            keyword = args.list_tests_by[1:]
+            list_testsuite_by(criteria, keyword)
+
+    if args.list_tags:
+        list_tags()
+
     if args.list_allclasses:
         args.list_modules = True
 
@@ -195,7 +405,7 @@ def main():
                     print e
                     pass
 
-    if args.run_tests or args.run_all_tests:
+    if args.run_tests or args.run_all_tests or args.run_tests_by:
         if not preflight_check():
             return 1
 
@@ -235,7 +445,11 @@ def main():
 
                 coverage_process_start = os.environ["COVERAGE_PROCESS_START"] = coveragerc
 
-        testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
+        if args.run_tests_by:
+            testslist = ts
+        else:
+            testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
+
         suite = unittest.TestSuite()
         loader = unittest.TestLoader()
         loader.sortTestMethodsUsing = None