add test for installing extensions in parallel

Author: boegel

test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py

@@ -30,7 +30,8 @@
 
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
-from easybuild.easyblocks.toy import EB_toy
+from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd
+from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.run import run_cmd
 
 
@@ -45,20 +46,55 @@ def extra_options():
         }
         return ExtensionEasyBlock.extra_options(extra_vars=extra_vars)
 
-    def run(self):
-        """Build toy extension."""
+    @property
+    def required_deps(self):
+        """Return list of required dependencies for this extension."""
+        deps = {
+            'bar': [],
+            'barbar': ['bar'],
+            'ls': [],
+        }
+        if self.name in deps:
+            return deps[self.name]
+        else:
+            raise EasyBuildError("Dependencies for %s are unknown!", self.name)
+
+    def run(self, *args, **kwargs):
+        """
+        Install toy extension.
+        """
         if self.src:
-            super(Toy_Extension, self).run(unpack_src=True)
-            EB_toy.configure_step(self.master, name=self.name)
             EB_toy.build_step(self.master, name=self.name, buildopts=self.cfg['buildopts'])
 
             if self.cfg['toy_ext_param']:
                 run_cmd(self.cfg['toy_ext_param'])
 
-            EB_toy.install_step(self.master, name=self.name)
-
             return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper(), self.name)
 
+    def prerun(self):
+        """
+        Prepare installation of toy extension.
+        """
+        if self.src:
+            super(Toy_Extension, self).run(unpack_src=True)
+            EB_toy.configure_step(self.master, name=self.name)
+
+    def run_async(self):
+        """
+        Install toy extension asynchronously.
+        """
+        if self.src:
+            cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts'])
+            self.async_cmd_start(cmd)
+        else:
+            self.async_cmd_info = False
+
+    def postrun(self):
+        """
+        Wrap up installation of toy extension.
+        """
+        EB_toy.install_step(self.master, name=self.name)
+
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for toy extensions."""
         self.log.info("Loaded modules: %s", self.modules_tool.list())

test/framework/sandbox/easybuild/easyblocks/t/toy.py

@@ -41,6 +41,19 @@
 from easybuild.tools.run import run_cmd
 
 
+def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts):
+    """
+    Compose command to build toy.
+    """
+
+    cmd = "%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s" % {
+        'name': name,
+        'prebuildopts': prebuildopts,
+        'buildopts': buildopts,
+    }
+    return cmd
+
+
 class EB_toy(ExtensionEasyBlock):
     """Support for building/installing toy."""
 
@@ -92,17 +105,13 @@ def configure_step(self, name=None):
 
     def build_step(self, name=None, buildopts=None):
         """Build toy."""
-
         if buildopts is None:
             buildopts = self.cfg['buildopts']
-
         if name is None:
             name = self.name
-        run_cmd('%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s' % {
-            'name': name,
-            'prebuildopts': self.cfg['prebuildopts'],
-            'buildopts': buildopts,
-        })
+
+        cmd = compose_toy_build_cmd(self.cfg, name, self.cfg['prebuildopts'], buildopts)
+        run_cmd(cmd)
 
     def install_step(self, name=None):
         """Install toy."""
@@ -118,11 +127,38 @@ def install_step(self, name=None):
         mkdir(libdir, parents=True)
         write_file(os.path.join(libdir, 'lib%s.a' % name), name.upper())
 
-    def run(self):
-        """Install toy as extension."""
+    @property
+    def required_deps(self):
+        """Return list of required dependencies for this extension."""
+        if self.name == 'toy':
+            return ['bar', 'barbar']
+        else:
+            raise EasyBuildError("Dependencies for %s are unknown!", self.name)
+
+    def prerun(self):
+        """
+        Prepare installation of toy as extension.
+        """
         super(EB_toy, self).run(unpack_src=True)
         self.configure_step()
+
+    def run(self):
+        """
+        Install toy as extension.
+        """
         self.build_step()
+
+    def run_async(self):
+        """
+        Asynchronous installation of toy as extension.
+        """
+        cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts'])
+        self.async_cmd_start(cmd)
+
+    def postrun(self):
+        """
+        Wrap up installation of toy as extension.
+        """
         self.install_step()
 
     def make_module_step(self, fake=False):

test/framework/toy_build.py

@@ -1777,6 +1777,45 @@ def test_module_only_extensions(self):
         self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True)
         self.assertTrue(os.path.exists(toy_mod))
 
+    def test_toy_exts_parallel(self):
+        """
+        Test parallel installation of extensions (--parallel-extensions-install)
+        """
+        topdir = os.path.abspath(os.path.dirname(__file__))
+        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+
+        toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+        if get_module_syntax() == 'Lua':
+            toy_mod += '.lua'
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        test_ec_txt = read_file(toy_ec)
+        test_ec_txt += '\n' + '\n'.join([
+            "exts_list = [",
+            "    ('ls'),",
+            "    ('bar', '0.0'),",
+            "    ('barbar', '0.0', {",
+            "        'start_dir': 'src',",
+            "    }),",
+            "    ('toy', '0.0'),",
+            "]",
+            "sanity_check_commands = ['barbar', 'toy']",
+            "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}",
+        ])
+        write_file(test_ec, test_ec_txt)
+
+        args = ['--parallel-extensions-install', '--experimental', '--force']
+        stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+        self.assertEqual(stderr, '')
+        expected_stdout = '\n'.join([
+            "== 0 out of 4 extensions installed (2 queued, 2 running: ls, bar)",
+            "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)",
+            "== 3 out of 4 extensions installed (0 queued, 1 running: toy)",
+            "== 4 out of 4 extensions installed (0 queued, 0 running: )",
+            '',
+        ])
+        self.assertEqual(stdout, expected_stdout)
+
     def test_backup_modules(self):
         """Test use of backing up of modules with --module-only."""