Merge pull request #3808 from ComputeCanada/fix_bash_shebang_for

add support for fix_bash_shebang_for

Author: boegel

easybuild/framework/easyblock.py

@@ -2485,7 +2485,7 @@ def package_step(self):
 
     def fix_shebang(self):
         """Fix shebang lines for specified files."""
-        for lang in ['perl', 'python']:
+        for lang in ['bash', 'perl', 'python']:
             shebang_regex = re.compile(r'^#![ ]*.*[/ ]%s.*' % lang)
             fix_shebang_for = self.cfg['fix_%s_shebang_for' % lang]
             if fix_shebang_for:

easybuild/framework/easyconfig/default.py

@@ -95,6 +95,8 @@
     'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD],
     'enhance_sanity_check': [False, "Indicate that additional sanity check commands & paths should enhance "
                              "the existin sanity check, not replace it", BUILD],
+    'fix_bash_shebang_for': [None, "List of files for which Bash shebang should be fixed "
+                                   "to '#!/usr/bin/env bash' (glob patterns supported)", BUILD],
     'fix_perl_shebang_for': [None, "List of files for which Perl shebang should be fixed "
                                    "to '#!/usr/bin/env perl' (glob patterns supported)", BUILD],
     'fix_python_shebang_for': [None, "List of files for which Python shebang should be fixed "

test/framework/toy_build.py

@@ -2951,6 +2951,7 @@ def test_fix_shebang(self):
             # copy of bin/toy to use in fix_python_shebang_for and fix_perl_shebang_for
             "    'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.python',",
             "    'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.perl',",
+            "    'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.sh',",
 
             # hardcoded path to bin/python
             "   'echo \"#!/usr/bin/python\\n# test\" > %(installdir)s/bin/t1.py',",
@@ -2987,9 +2988,26 @@ def test_fix_shebang(self):
             # shebang bash
             "   'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b2.sh',",
 
+            # tests for bash shebang
+            # hardcoded path to bin/bash
+            "   'echo \"#!/bin/bash\\n# test\" > %(installdir)s/bin/t1.sh',",
+            # hardcoded path to usr/bin/bash
+            "   'echo \"#!/usr/bin/bash\\n# test\" > %(installdir)s/bin/t2.sh',",
+            # already OK, should remain the same
+            "   'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/t3.sh',",
+            # shebang with space, should strip the space
+            "   'echo \"#! /usr/bin/env bash\\n# test\" > %(installdir)s/bin/t4.sh',",
+            # no shebang sh
+            "   'echo \"# test\" > %(installdir)s/bin/t5.sh',",
+            # shebang python
+            "   'echo \"#!/usr/bin/env python\\n# test\" > %(installdir)s/bin/b1.py',",
+            # shebang perl
+            "   'echo \"#!/usr/bin/env perl\\n# test\" > %(installdir)s/bin/b1.pl',",
+
             "]",
-            "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']",
-            "fix_perl_shebang_for = ['bin/*.pl', 'bin/b2.sh', 'bin/toy.perl']",
+            "fix_python_shebang_for = ['bin/t1.py', 'bin/t*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']",
+            "fix_perl_shebang_for = ['bin/t*.pl', 'bin/b2.sh', 'bin/toy.perl']",
+            "fix_bash_shebang_for = ['bin/t*.sh', 'bin/b1.py', 'bin/b1.pl', 'bin/toy.sh']",
         ])
         write_file(test_ec, test_ec_txt)
         self.test_toy_build(ec_file=test_ec, raise_error=True)
@@ -2998,36 +3016,31 @@ def test_fix_shebang(self):
 
         # bin/toy and bin/toy2 should *not* be patched, since they're binary files
         toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb')
-        for fn in ['toy.perl', 'toy.python']:
+        for fn in ['toy.sh', 'toy.perl', 'toy.python']:
             fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb')
             # no shebang added
             self.assertFalse(fn_txt.startswith(b"#!/"))
             # exact same file as original binary (untouched)
             self.assertEqual(toy_txt, fn_txt)
 
+        regexes = {}
         # no re.M, this should match at start of file!
-        py_shebang_regex = re.compile(r'^#!/usr/bin/env python\n# test$')
-        for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']:
-            pybin_path = os.path.join(toy_bindir, pybin)
-            pybin_txt = read_file(pybin_path)
-            self.assertTrue(py_shebang_regex.match(pybin_txt),
-                            "Pattern '%s' found in %s: %s" % (py_shebang_regex.pattern, pybin_path, pybin_txt))
-
-        # no re.M, this should match at start of file!
-        perl_shebang_regex = re.compile(r'^#!/usr/bin/env perl\n# test$')
-        for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']:
-            perlbin_path = os.path.join(toy_bindir, perlbin)
-            perlbin_txt = read_file(perlbin_path)
-            self.assertTrue(perl_shebang_regex.match(perlbin_txt),
-                            "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt))
-
-        # There are 2 bash files which shouldn't be influenced by fix_shebang
-        bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$')
-        for bashbin in ['b1.sh', 'b2.sh']:
-            bashbin_path = os.path.join(toy_bindir, bashbin)
-            bashbin_txt = read_file(bashbin_path)
-            self.assertTrue(bash_shebang_regex.match(bashbin_txt),
-                            "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt))
+        regexes['py'] = re.compile(r'^#!/usr/bin/env python\n# test$')
+        regexes['pl'] = re.compile(r'^#!/usr/bin/env perl\n# test$')
+        regexes['sh'] = re.compile(r'^#!/usr/bin/env bash\n# test$')
+
+        # all scripts should have a shebang that matches their extension
+        scripts = {}
+        scripts['py'] = ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py', 'b1.py']
+        scripts['pl'] = ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl', 'b1.pl']
+        scripts['sh'] = ['t1.sh', 't2.sh', 't3.sh', 't4.sh', 't5.sh', 'b1.sh', 'b2.sh']
+
+        for ext in ['sh', 'pl', 'py']:
+            for script in scripts[ext]:
+                bin_path = os.path.join(toy_bindir, script)
+                bin_txt = read_file(bin_path)
+                self.assertTrue(regexes[ext].match(bin_txt),
+                                "Pattern '%s' found in %s: %s" % (regexes[ext].pattern, bin_path, bin_txt))
 
         # now test with a custom env command
         extra_args = ['--env-for-shebang=/usr/bin/env -S']
@@ -3037,36 +3050,30 @@ def test_fix_shebang(self):
 
         # bin/toy and bin/toy2 should *not* be patched, since they're binary files
         toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb')
-        for fn in ['toy.perl', 'toy.python']:
+        for fn in ['toy.sh', 'toy.perl', 'toy.python']:
             fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb')
             # no shebang added
             self.assertFalse(fn_txt.startswith(b"#!/"))
             # exact same file as original binary (untouched)
             self.assertEqual(toy_txt, fn_txt)
 
+        regexes_S = {}
         # no re.M, this should match at start of file!
-        py_shebang_regex = re.compile(r'^#!/usr/bin/env -S python\n# test$')
-        for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']:
-            pybin_path = os.path.join(toy_bindir, pybin)
-            pybin_txt = read_file(pybin_path)
-            self.assertTrue(py_shebang_regex.match(pybin_txt),
-                            "Pattern '%s' found in %s: %s" % (py_shebang_regex.pattern, pybin_path, pybin_txt))
-
-        # no re.M, this should match at start of file!
-        perl_shebang_regex = re.compile(r'^#!/usr/bin/env -S perl\n# test$')
-        for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']:
-            perlbin_path = os.path.join(toy_bindir, perlbin)
-            perlbin_txt = read_file(perlbin_path)
-            self.assertTrue(perl_shebang_regex.match(perlbin_txt),
-                            "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt))
-
-        # There are 2 bash files which shouldn't be influenced by fix_shebang
-        bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$')
-        for bashbin in ['b1.sh', 'b2.sh']:
-            bashbin_path = os.path.join(toy_bindir, bashbin)
-            bashbin_txt = read_file(bashbin_path)
-            self.assertTrue(bash_shebang_regex.match(bashbin_txt),
-                            "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt))
+        regexes_S['py'] = re.compile(r'^#!/usr/bin/env -S python\n# test$')
+        regexes_S['pl'] = re.compile(r'^#!/usr/bin/env -S perl\n# test$')
+        regexes_S['sh'] = re.compile(r'^#!/usr/bin/env -S bash\n# test$')
+
+        for ext in ['sh', 'pl', 'py']:
+            for script in scripts[ext]:
+                bin_path = os.path.join(toy_bindir, script)
+                bin_txt = read_file(bin_path)
+                # the scripts b1.py, b1.pl, b1.sh, b2.sh should keep their original shebang
+                if script.startswith('b'):
+                    self.assertTrue(regexes[ext].match(bin_txt),
+                                    "Pattern '%s' found in %s: %s" % (regexes[ext].pattern, bin_path, bin_txt))
+                else:
+                    self.assertTrue(regexes_S[ext].match(bin_txt),
+                                    "Pattern '%s' found in %s: %s" % (regexes_S[ext].pattern, bin_path, bin_txt))
 
     def test_toy_system_toolchain_alias(self):
         """Test use of 'system' toolchain alias."""