keep ModuleTool.mod_paths in sync with $MODULEPATH in ModuleTool.load()

easybuild/framework/easyblock.py

@@ -1871,10 +1871,14 @@ def load_fake_module(self, purge=False, extra_modules=None, verbose=False):
 
         # create fake module
         fake_mod_path = self.make_module_step(fake=True)
+        full_fake_mod_path = os.path.join(fake_mod_path, self.mod_subdir)
+
+        # indicate that path to fake module should get absolute priority
+        mod_paths = [(full_fake_mod_path, 10000)]
 
         # load fake module
-        self.modules_tool.prepend_module_path(os.path.join(fake_mod_path, self.mod_subdir), priority=10000)
-        self.load_module(purge=purge, extra_modules=extra_modules, verbose=verbose)
+        self.load_module(purge=purge, extra_modules=extra_modules, mod_paths=mod_paths, verbose=verbose)
+
         return (fake_mod_path, env)
 
     def clean_up_fake_module(self, fake_mod_data):

easybuild/tools/modules.py

@@ -1104,17 +1104,24 @@ def load(self, modules, mod_paths=None, purge=False, init_env=None, allow_reload
             # restore initial environment if provided
             if init_env is None:
                 raise EasyBuildError("Initial environment required when purging before loading, but not available")
-            else:
-                restore_env(init_env)
 
-            # make sure $MODULEPATH is set correctly after purging
+            restore_env(init_env)
+
+            # sync mod_paths class variable with $MODULEPATH environment variable after resetting environment
+            self.mod_paths = None
             self.check_module_path()
 
         # extend $MODULEPATH if needed
         for mod_path in mod_paths:
+            priority = None
+            if isinstance(mod_path, tuple) and len(mod_path) == 2:
+                mod_path, priority = mod_path
+            elif not isinstance(mod_path, str):
+                raise EasyBuildError(f"Incorrect mod_paths entry encountered when loading module(s): {mod_path}")
+
             full_mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'), mod_path)
             if os.path.exists(full_mod_path):
-                self.prepend_module_path(full_mod_path)
+                self.prepend_module_path(full_mod_path, priority=priority)
 
         loaded_modules = self.loaded_modules()
         for mod in modules:

test/framework/easyblock.py

@@ -31,6 +31,7 @@
 @author: Jan Andre Reuter (Juelich Supercomputing Centre)
 """
 import copy
+import fileinput
 import os
 import re
 import shutil
@@ -218,6 +219,46 @@ def test_load_module(self):
         eb.close_log()
         os.remove(eb.logfile)
 
+        # test HMNS module load when conflicting dependencies are available in both Core and
+        # toolchain-specific modulepaths
+        # see also https://github.com/easybuilders/easybuild-framework/issues/4986
+        test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                     'easyconfigs', 'test_ecs')
+        os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
+        build_options = {
+            'generate_devel_module': True,  # go through EasyBlock.fake_module_environment()
+            'robot_path': [test_ecs_path],
+        }
+        init_config(build_options=build_options)
+
+        # setup pre-built test modules under test install path
+        mod_prefix = os.path.join(self.test_installpath, 'modules', 'all')
+        mkdir(mod_prefix, parents=True)
+        for mod_subdir in ['Core', 'Compiler']:
+            src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                        'modules', 'HierarchicalMNS', mod_subdir)
+            copy_dir(src_mod_path, os.path.join(mod_prefix, mod_subdir))
+
+        # tweak use statements in toolchain module to ensure correct paths
+        modfile = os.path.join(mod_prefix, 'Core', 'GCCcore', '12.3.0')
+        for line in fileinput.input(modfile, inplace=1):
+            line = re.sub(r"(module\s*use\s*)/tmp/modules/all",
+                          r"\1%s/modules/all" % self.test_installpath,
+                          line)
+            sys.stdout.write(line)
+
+        test_eb_file = os.path.join(test_ecs_path, 'g', 'GLib', 'GLib-2.77.1-GCCcore-12.3.0.eb')
+        eb = EasyBlock(EasyConfig(test_eb_file))
+
+        self.reset_modulepath([os.path.join(mod_prefix)])
+
+        with self.mocked_stdout_stderr():
+            eb.check_readiness_step()
+            eb.make_builddir()
+            eb.prepare_step()
+            eb.make_module_step()
+            eb.load_module()
+
     def test_fake_module_load(self):
         """Testcase for fake module load"""
         self.contents = '\n'.join([

test/framework/easyconfigs/test_ecs/g/GCCcore/GCCcore-12.3.0.eb

@@ -0,0 +1,23 @@
+# should be EB_GCC, but OK for testing purposes
+easyblock = 'EB_toy'
+
+name = "GCCcore"
+version = '12.3.0'
+
+homepage = 'http://gcc.gnu.org/'
+description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada,
+ as well as libraries for these languages (libstdc++, libgcj,...)."""
+
+toolchain = SYSTEM
+
+source_urls = [
+    'http://ftpmirror.gnu.org/%(namelower)s/%(namelower)s-%(version)s',  # GCC auto-resolving HTTP mirror
+]
+
+#gcc_name = 'GCC'
+
+sources = [
+    SOURCELOWER_TAR_BZ2,
+]
+
+moduleclass = 'compiler'

test/framework/easyconfigs/test_ecs/g/GLib/GLib-2.77.1-GCCcore-12.3.0.eb

@@ -0,0 +1,19 @@
+easyblock = 'EB_toy'
+
+name = 'GLib'
+version = '2.77.1'
+
+homepage = 'https://www.gtk.org/'
+description = """GLib is one of the base libraries of the GTK+ project"""
+
+toolchain = {'name': 'GCCcore', 'version': '12.3.0'}
+
+source_urls = [FTPGNOME_SOURCE]
+sources = [SOURCELOWER_TAR_XZ]
+
+dependencies = [
+    ('gettext', '0.21.1'),
+    ('util-linux', '2.39'),
+]
+
+moduleclass = 'vis'

test/framework/easyconfigs/test_ecs/g/gettext/gettext-0.21.1-GCCcore-12.3.0.eb

@@ -0,0 +1,20 @@
+easyblock = 'EB_toy'
+
+name = 'gettext'
+version = '0.21.1'
+
+homepage = 'https://www.gnu.org/software/gettext/'
+description = """GNU 'gettext' is an important step for the GNU Translation Project, as it is an asset on which we may
+build many other steps. This package offers to programmers, translators, and even users, a well integrated set of tools
+and documentation"""
+
+toolchain = {'name': 'GCCcore', 'version': '12.3.0'}
+
+source_urls = [GNU_SOURCE]
+sources = [SOURCE_TAR_GZ]
+
+dependencies = [
+    ('ncurses', '6.4'),
+]
+
+moduleclass = 'tools'

test/framework/easyconfigs/test_ecs/n/ncurses/ncurses-6.4-GCCcore-12.3.0.eb

@@ -0,0 +1,19 @@
+easyblock = 'EB_toy'
+
+name = 'ncurses'
+version = '6.4'
+
+homepage = 'https://www.gnu.org/software/ncurses/'
+description = """
+ The Ncurses (new curses) library is a free software emulation of curses in
+ System V Release 4.0, and more. It uses Terminfo format, supports pads and
+ color and multiple highlights and forms characters and function-key mapping,
+ and has all the other SYSV-curses enhancements over BSD Curses.
+"""
+
+toolchain = {'name': 'GCCcore', 'version': '12.3.0'}
+
+source_urls = [GNU_SOURCE]
+sources = [SOURCE_TAR_GZ]
+
+moduleclass = 'devel'

test/framework/easyconfigs/test_ecs/u/util-linux/util-linux-2.39-GCCcore-12.3.0.eb

@@ -0,0 +1,19 @@
+easyblock = 'EB_toy'
+
+name = 'util-linux'
+version = '2.39'
+
+homepage = 'https://www.kernel.org/pub/linux/utils/util-linux'
+
+description = "Set of Linux utilities"
+
+toolchain = {'name': 'GCCcore', 'version': '12.3.0'}
+
+source_urls = ['%s/v%%(version_major_minor)s' % homepage]
+sources = [SOURCELOWER_TAR_GZ]
+
+dependencies = [
+    ('ncurses', '6.4'),
+]
+
+moduleclass = 'tools'

test/framework/filetools.py

@@ -2641,7 +2641,7 @@ def test_index_functions(self):
         # test with specified path with and without trailing '/'s
         for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
             index = ft.create_index(path)
-            self.assertEqual(len(index), 95)
+            self.assertEqual(len(index), 100)
 
             expected = [
                 os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),
@@ -2691,7 +2691,7 @@ def test_index_functions(self):
         regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir)
         self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))
 
-        self.assertEqual(len(index), 26)
+        self.assertEqual(len(index), 29)
         for fn in expected:
             self.assertIn(fn, index)
 
@@ -2721,7 +2721,7 @@ def test_index_functions(self):
         regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir)
         self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))
 
-        self.assertEqual(len(index), 26)
+        self.assertEqual(len(index), 29)
         for fn in expected:
             self.assertIn(fn, index)
 

test/framework/modules.py

@@ -55,7 +55,7 @@
 
 
 # number of modules included for testing purposes
-TEST_MODULES_COUNT = 111
+TEST_MODULES_COUNT = 118
 
 
 class ModulesTest(EnhancedTestCase):
@@ -429,7 +429,7 @@ def test_load(self):
         ms = self.modtool.available()
         # exclude modules not on the top level of a hierarchy
         ms = [m for m in ms if not (m.startswith('Core') or m.startswith('Compiler/') or m.startswith('MPI/') or
-                                    m.startswith('CategorizedHMNS'))]
+                                    m.startswith('CategorizedHMNS') or m.startswith('HierarchicalMNS'))]
 
         for m in ms:
             self.modtool.load([m])
@@ -480,6 +480,21 @@ def test_load(self):
         self.assertEqual(os.environ.get('EBROOTGCC'), None)
         self.assertNotEqual(loaded_modules[-1], 'GCC/6.4.0-2.28')
 
+        # test loading of module when particular module path has priority over others
+        if isinstance(self.modtool, Lmod):
+            mod_paths = [
+                os.path.join(self.test_prefix, 'one'),
+                (os.path.join(self.test_prefix, 'two'), 10000),
+                os.path.join(self.test_prefix, 'three'),
+            ]
+            for mod_path in mod_paths:
+                if isinstance(mod_path, tuple):
+                    mod_path = mod_path[0]
+                mod_file = os.path.join(mod_path, 'test', '1.2.3.lua')
+                write_file(mod_file, 'setenv("TEST_PARENT_DIR", "%s")' % os.path.basename(mod_path))
+            self.modtool.load(['test/1.2.3'], mod_paths=mod_paths)
+            self.assertEqual(os.getenv('TEST_PARENT_DIR'), 'two')
+
     def test_show(self):
         """Test for ModulesTool.show method."""
 

test/framework/modules/HierarchicalMNS/Compiler/GCCcore/12.3.0/GLib/2.77.1

@@ -0,0 +1,32 @@
+#%Module
+proc ModulesHelp { } {
+    puts stderr {
+
+Description
+===========
+GLib is one of the base libraries of the GTK+ project
+
+
+More information
+================
+ - Homepage: https://www.gtk.org/
+    }
+}
+
+module-whatis {Description: GLib is one of the base libraries of the GTK+ project}
+module-whatis {Homepage: https://www.gtk.org/}
+module-whatis {URL: https://www.gtk.org/}
+
+set root /tmp/software/GLib/2.77.1-GCCcore-12.3.0
+
+conflict GLib
+
+module load gettext/0.21.1
+
+module load util-linux/2.39
+
+setenv	EBROOTGLIB		"$root"
+setenv	EBVERSIONGLIB		"2.77.1"
+setenv	EBDEVELGLIB		"$root/easybuild/Compiler-GCCcore-12.3.0-GLib-2.77.1-easybuild-devel"
+
+# Built with EasyBuild version 5.1.2.dev0-r5819168038d7bd74810f38f75acfced9fb41870e

test/framework/modules/HierarchicalMNS/Compiler/GCCcore/12.3.0/gettext/0.21.1

@@ -0,0 +1,34 @@
+#%Module
+proc ModulesHelp { } {
+    puts stderr {
+
+Description
+===========
+GNU 'gettext' is an important step for the GNU Translation Project, as it is an asset on which we may
+build many other steps. This package offers to programmers, translators, and even users, a well integrated set of tools
+and documentation
+
+
+More information
+================
+ - Homepage: https://www.gnu.org/software/gettext/
+    }
+}
+
+module-whatis {Description: GNU 'gettext' is an important step for the GNU Translation Project, as it is an asset on which we may
+build many other steps. This package offers to programmers, translators, and even users, a well integrated set of tools
+and documentation}
+module-whatis {Homepage: https://www.gnu.org/software/gettext/}
+module-whatis {URL: https://www.gnu.org/software/gettext/}
+
+set root /tmp/software/gettext/0.21.1-GCCcore-12.3.0
+
+conflict gettext
+
+module load ncurses/6.4
+
+setenv	EBROOTGETTEXT		"$root"
+setenv	EBVERSIONGETTEXT		"0.21.1"
+setenv	EBDEVELGETTEXT		"$root/easybuild/Compiler-GCCcore-12.3.0-gettext-0.21.1-easybuild-devel"
+
+# Built with EasyBuild version 5.1.2.dev0-r5819168038d7bd74810f38f75acfced9fb41870e

test/framework/modules/HierarchicalMNS/Compiler/GCCcore/12.3.0/ncurses/6.4

@@ -0,0 +1,36 @@
+#%Module
+proc ModulesHelp { } {
+    puts stderr {
+
+Description
+===========
+The Ncurses (new curses) library is a free software emulation of curses in
+ System V Release 4.0, and more. It uses Terminfo format, supports pads and
+ color and multiple highlights and forms characters and function-key mapping,
+ and has all the other SYSV-curses enhancements over BSD Curses.
+
+
+More information
+================
+ - Homepage: https://www.gnu.org/software/ncurses/
+    }
+}
+
+module-whatis {Description:
+ The Ncurses (new curses) library is a free software emulation of curses in
+ System V Release 4.0, and more. It uses Terminfo format, supports pads and
+ color and multiple highlights and forms characters and function-key mapping,
+ and has all the other SYSV-curses enhancements over BSD Curses.
+}
+module-whatis {Homepage: https://www.gnu.org/software/ncurses/}
+module-whatis {URL: https://www.gnu.org/software/ncurses/}
+
+set root /tmp/software/ncurses/6.4-GCCcore-12.3.0
+
+conflict ncurses
+
+setenv	EBROOTNCURSES		"$root"
+setenv	EBVERSIONNCURSES		"6.4"
+setenv	EBDEVELNCURSES		"$root/easybuild/Compiler-GCCcore-12.3.0-ncurses-6.4-easybuild-devel"
+
+# Built with EasyBuild version 5.1.2.dev0-r5819168038d7bd74810f38f75acfced9fb41870e