Merge pull request #3206 from sarahmarshy/examples-test-filters

[Exporter/compile tests] Examples test filters

Author: sg-

tools/test/examples/examples.py

@@ -12,17 +12,24 @@
 sys.path.insert(0, ROOT)
 
 from tools.utils import argparse_force_uppercase_type
+from tools.utils import argparse_many
+from tools.build_api import get_mbed_official_release
 import examples_lib as lib
 from examples_lib import SUPPORTED_TOOLCHAINS, SUPPORTED_IDES
 
-EXAMPLES = json.load(open(os.path.join(os.path.dirname(__file__),
-                                       "examples.json")))
-
-
 def main():
     """Entry point"""
+
+    official_targets = get_mbed_official_release("5")
+    official_target_names = [x[0] for x in official_targets]
+
+
     parser = ArgumentParser()
     parser.add_argument("-c", dest="config", default="examples.json")
+    parser.add_argument("-e", "--example",
+                        help=("filter the examples used in the script"),
+                        type=argparse_many(lambda x: x),
+                        default=[])
     subparsers = parser.add_subparsers()
     import_cmd = subparsers.add_parser("import")
     import_cmd.set_defaults(fn=do_import)
@@ -39,60 +46,81 @@ def main():
         "toolchains", nargs="*", default=SUPPORTED_TOOLCHAINS,
         type=argparse_force_uppercase_type(SUPPORTED_TOOLCHAINS,
                                            "toolchain")),
+    compile_cmd.add_argument("-m", "--mcu",
+                             help=("build for the given MCU (%s)" %
+                                ', '.join(official_target_names)),
+                            metavar="MCU",
+                            type=argparse_many(
+                                argparse_force_uppercase_type(
+                                    official_target_names, "MCU")),
+                            default=official_target_names)
     export_cmd = subparsers.add_parser("export")
     export_cmd.set_defaults(fn=do_export),
     export_cmd.add_argument(
         "ide", nargs="*", default=SUPPORTED_IDES,
         type=argparse_force_uppercase_type(SUPPORTED_IDES,
                                            "ide"))
+    export_cmd.add_argument("-m", "--mcu",
+                             help=("build for the given MCU (%s)" %
+                                ', '.join(official_target_names)),
+                            metavar="MCU",
+                            type=argparse_many(
+                                argparse_force_uppercase_type(
+                                    official_target_names, "MCU")),
+                            default=official_target_names)
     args = parser.parse_args()
     config = json.load(open(os.path.join(os.path.dirname(__file__),
                                args.config)))
-    return args.fn(args, config)
+
+    all_examples = []
+    for example in config['examples']:
+        all_examples = all_examples + [basename(x['repo']) for x in lib.get_repo_list(example)]
+    examples = [x for x in all_examples if x in args.example] if args.example else all_examples
+    return args.fn(args, config, examples)
 
 
-def do_export(args, config):
+def do_export(args, config, examples):
     """Do export and build step"""
     results = {}
-    results = lib.export_repos(config, args.ide)
+    results = lib.export_repos(config, args.ide, args.mcu, examples)
 
     lib.print_summary(results, export=True)
     failures = lib.get_num_failures(results, export=True)
     print("Number of failures = %d" % failures)
     return failures
 
 
-def do_import(_, config):
+def do_import(_, config, examples):
     """Do the import step of this process"""
-    lib.source_repos(config)
+    lib.source_repos(config, examples)
     return 0
 
 
-def do_clone(_, config):
+def do_clone(_, config, examples):
     """Do the clone step of this process"""
-    lib.clone_repos(config)
+    lib.clone_repos(config, examples)
     return 0
 
 
-def do_deploy(_, config):
+def do_deploy(_, config, examples):
     """Do the deploy step of this process"""
-    lib.deploy_repos(config)
+    lib.deploy_repos(config, examples)
     return 0
 
 
-def do_compile(args, config):
+def do_compile(args, config, examples):
     """Do the compile step"""
     results = {}
-    results = lib.compile_repos(config, args.toolchains)
+    results = lib.compile_repos(config, args.toolchains, args.mcu, examples)
 
     lib.print_summary(results)
     failures = lib.get_num_failures(results)
     print("Number of failures = %d" % failures)
     return failures 
 
-def do_versionning(args, config):
+def do_versionning(args, config, examples):
     """ Test update the mbed-os to the version specified by the tag """
-    lib.update_mbedos_version(config, args.tag)        
+    lib.update_mbedos_version(config, args.tag, examples)
     return 0
 
 

tools/test/examples/examples_lib.py

@@ -75,52 +75,46 @@ def print_summary(results, export=False):
 
     print("#")
     print("#"*80)
-    
 
-def target_cross_toolchain(allowed_toolchains,
-                           features=[], targets=[]):
+def valid_choices(allowed_choices, all_choices):
+    if len(allowed_choices) > 0:
+        return [t for t in all_choices if t in allowed_choices]
+    else:
+        return all_choices
+
+
+def target_cross_toolchain(allowed_targets, allowed_toolchains, features=[]):
     """Generate pairs of target and toolchains
 
     Args:
+    allowed_targets - a list of all possible targets
     allowed_toolchains - a list of all possible toolchains
 
     Kwargs:
     features - the features that must be in the features array of a
                target
-    targets - a list of available targets
     """
-    if len(targets) == 0:
-        targets=TARGET_MAP.keys()
-        
-    for target, toolchains in get_mbed_official_release("5"):
-        for toolchain in toolchains:
-            if (toolchain in allowed_toolchains and
-                target in targets and
-                all(feature in TARGET_MAP[target].features
-                    for feature in features)):
+    for target in allowed_targets:
+        for toolchain in allowed_toolchains:
+            if all(feature in TARGET_MAP[target].features
+                    for feature in features):
                 yield target, toolchain
 
 
-def target_cross_ide(allowed_ides,
-                    features=[], targets=[]):
+def target_cross_ide(allowed_targets, allowed_ides, features=[]):
     """Generate pairs of target and ides
 
     Args:
+    allowed_targets - a list of all possible targets
     allowed_ides - a list of all possible IDEs
 
     Kwargs:
     features - the features that must be in the features array of a
                target
-    targets - a list of available targets
     """
-    if len(targets) == 0:
-        targets=TARGET_MAP.keys()
-
-    for target, toolchains in get_mbed_official_release("5"):
+    for target in allowed_targets:
         for ide in allowed_ides:
-            if (EXPORTERS[ide].TOOLCHAIN in toolchains and
-                target in EXPORTERS[ide].TARGETS and
-                target in targets and
+            if (target in EXPORTERS[ide].TARGETS and
                 all(feature in TARGET_MAP[target].features
                     for feature in features)):
                 yield target, ide
@@ -154,7 +148,7 @@ def get_repo_list(example):
     return repos
 
 
-def source_repos(config):
+def source_repos(config, examples):
     """ Imports each of the repos and its dependencies (.lib files) associated
         with the specific examples name from the json config file. Note if
         there is already a clone of the repo then it will first be removed to
@@ -167,13 +161,14 @@ def source_repos(config):
     for example in config['examples']:
         for repo_info in get_repo_list(example):
             name = basename(repo_info['repo'])
-            if os.path.exists(name):
-                print("'%s' example directory already exists. Deleting..." % name)
-                rmtree(name)
-        
-            subprocess.call(["mbed-cli", "import", repo_info['repo']])
+            if name in examples:
+                if os.path.exists(name):
+                    print("'%s' example directory already exists. Deleting..." % name)
+                    rmtree(name)
 
-def clone_repos(config):
+                subprocess.call(["mbed-cli", "import", repo_info['repo']])
+
+def clone_repos(config, examples):
     """ Clones each of the repos associated with the specific examples name from the
         json config file. Note if there is already a clone of the repo then it will first
         be removed to ensure a clean, up to date cloning.
@@ -185,13 +180,14 @@ def clone_repos(config):
     for example in config['examples']:
         for repo_info in get_repo_list(example):
             name = basename(repo_info['repo'])
-            if os.path.exists(name):
-                print("'%s' example directory already exists. Deleting..." % name)
-                rmtree(name)
+            if name in examples:
+                if os.path.exists(name):
+                    print("'%s' example directory already exists. Deleting..." % name)
+                    rmtree(name)
 
-            subprocess.call([repo_info['type'], "clone", repo_info['repo']])
+                subprocess.call([repo_info['type'], "clone", repo_info['repo']])
 
-def deploy_repos(config):
+def deploy_repos(config, examples):
     """ If the example directory exists as provided by the json config file,
         pull in the examples dependencies by using `mbed-cli deploy`.
     Args:
@@ -202,13 +198,13 @@ def deploy_repos(config):
     for example in config['examples']:
         for repo_info in get_repo_list(example):
             name = basename(repo_info['repo'])
-
-            if os.path.exists(name):
-                os.chdir(name)
-                subprocess.call(["mbed-cli", "deploy"])
-                os.chdir("..")
-            else:
-                print("'%s' example directory doesn't exist. Skipping..." % name)
+            if name in examples:
+                if os.path.exists(name):
+                    os.chdir(name)
+                    subprocess.call(["mbed-cli", "deploy"])
+                    os.chdir("..")
+                else:
+                    print("'%s' example directory doesn't exist. Skipping..." % name)
 
 
 def get_num_failures(results, export=False):
@@ -228,35 +224,36 @@ def get_num_failures(results, export=False):
 
     return num_failures
 
-
-def export_repos(config, ides):
+def export_repos(config, ides, targets, examples):
     """Exports and builds combinations of example programs, targets and IDEs.
 
-           The results are returned in a [key: value] dictionary format:
-           Where key = The example name from the json config file
-                 value = a list containing: pass_status, successes, export failures, build_failures,
-                 and build_skips
-
-                 where pass_status = The overall pass status for the export of the full
-                                     set of example programs comprising the example suite.
-                                     (IE they must build and export)
-                                     True if all examples pass, false otherwise
-                       successes = list of examples that exported and built (if possible)
-                                   If the exporter has no build functionality, then it is a pass
-                                   if exported
-                       export_failures = list of examples that failed to export.
-                       build_failures = list of examples that failed to build
-                       build_skips = list of examples that cannot build
-
-                       Both successes and failures contain the example name, target and IDE
-
-        Args:
-        config - the json object imported from the file.
-        ides - List of IDES to export to
-        """
+        The results are returned in a [key: value] dictionary format:
+            Where key = The example name from the json config file
+            value = a list containing: pass_status, successes, export failures, build_failures,
+            and build_skips
+
+            where pass_status = The overall pass status for the export of the full
+            set of example programs comprising the example suite.
+            IE they must build and export) True if all examples pass, false otherwise
+            successes = list of examples that exported and built (if possible)
+            If the exporter has no build functionality, then it is a pass
+            if exported
+            export_failures = list of examples that failed to export.
+            build_failures = list of examples that failed to build
+            build_skips = list of examples that cannot build
+
+            Both successes and failures contain the example name, target and IDE
+
+            Args:
+            config - the json object imported from the file.
+            ides - List of IDES to export to
+    """
     results = {}
     print("\nExporting example repos....\n")
     for example in config['examples']:
+        if example['name'] not in examples:
+            continue
+
         export_failures = []
         build_failures = []
         build_skips = []
@@ -269,9 +266,9 @@ def export_repos(config, ides):
                 os.chdir(example_project_name)
                 # Check that the target, IDE, and features combinations are valid and return a
                 # list of valid combinations to work through
-                for target, ide in target_cross_ide(ides,
-                                                    example['features'],
-                                                    example['targets']):
+                for target, ide in target_cross_ide(valid_choices(example['targets'], targets),
+                                                    valid_choices(example['exporters'], ides),
+                                                    example['features']):
                     example_name = "{} {} {}".format(example_project_name, target,
                                                      ide)
                     def status(message):
@@ -311,7 +308,7 @@ def status(message):
     return results
 
 
-def compile_repos(config, toolchains):
+def compile_repos(config, toolchains, targets, examples):
     """Compiles combinations of example programs, targets and compile chains.
        
        The results are returned in a [key: value] dictionary format:
@@ -334,23 +331,23 @@ def compile_repos(config, toolchains):
     """
     results = {}
     print("\nCompiling example repos....\n")
-    for example in config['examples']:        
+    for example in config['examples']:
+        if example['name'] not in examples:
+            continue
         failures = []
         successes = []
         compiled = True
         pass_status = True
         if example['compile']:
-            if len(example['toolchains']) > 0:
-                toolchains = example['toolchains']
-            
             for repo_info in get_repo_list(example):
                 name = basename(repo_info['repo'])
                 os.chdir(name)
 
                 # Check that the target, toolchain and features combinations are valid and return a 
                 # list of valid combinations to work through
-                for target, toolchain in target_cross_toolchain(toolchains,
-                                                                example['features'], example['targets']):
+                for target, toolchain in target_cross_toolchain(valid_choices(example['targets'], targets),
+                                                                valid_choices(example['toolchains'], toolchains),
+                                                                example['features']):
                     proc = subprocess.Popen(["mbed-cli", "compile", "-t", toolchain,
                                              "-m", target, "--silent"])
                     proc.wait()
@@ -372,7 +369,7 @@ def compile_repos(config, toolchains):
     return results
 
 
-def update_mbedos_version(config, tag):
+def update_mbedos_version(config, tag, examples):
     """ For each example repo identified in the config json object, update the version of 
         mbed-os to that specified by the supplied GitHub tag. This function assumes that each
         example repo has already been cloned.
@@ -384,6 +381,8 @@ def update_mbedos_version(config, tag):
     """
     print("Updating mbed-os in examples to version %s\n" % tag)
     for example in config['examples']:
+        if example['name'] not in examples:
+            continue
         for repo_info in get_repo_list(example):
             update_dir =  basename(repo_info['repo']) + "/mbed-os"
             print("\nChanging dir to %s\n" % update_dir)