Fix tests to work with FIPS enabled

.gitignore

@@ -34,7 +34,7 @@ venv.bak/
 
 
 # SCons files
-.sconsign.*
+.sconsign*
 
 # Tool output
 .coverage

CHANGES.txt

@@ -9,6 +9,12 @@ NOTE: The 4.2.0 Release of SCons will deprecate Python 3.5 Support. Python 3.5 s
 
 RELEASE  VERSION/DATE TO BE FILLED IN LATER
 
+  From Jacob Cassagnol:
+    - Default hash algorithm check updated for Scons FIPS compliance. Now checks for hash viability
+      first and then walks the tree to use the first viable hash as the default one. This typically
+      selects SHA1 on FIPS-enabled systems as the new default instead of MD5, unless SHA1 has also
+      been disabled by security policy, at which point SCons selects SHA256 as the default.
+
   From Joseph Brill:
     - Fix MSVS tests (vs-N.N-exec.py) for MSVS 6.0, 7.0, and 7.1 (import missing module).
     - Add support for Visual Studio 2022 (release and prerelease).

SCons/Node/NodeTests.py

@@ -592,7 +592,7 @@ def my_contents(obj):
             node = SCons.Node.Node()
             node._func_get_contents = 4
             result = node.get_csig()
-            assert result == '550a141f12de6341fba65b0ad0433500', result
+            assert result == '550a141f12de6341fba65b0ad0433500' or result == '9a3e61b6bcc8abec08f195526c3132d5a4a98cc0' or result == '3538a1ef2e113da64249eea7bd068b585ec7ce5df73b2d1e319d8c9bf47eb314', result
         finally:
             SCons.Node.Node.NodeInfo = SCons.Node.NodeInfoBase
 
@@ -609,7 +609,7 @@ def my_contents(obj):
             node = SCons.Node.Node()
             node._func_get_contents = 4
             result = node.get_cachedir_csig()
-            assert result == '15de21c670ae7c3f6f3f1f37029303c9', result
+            assert result == '15de21c670ae7c3f6f3f1f37029303c9' or result == 'cfa1150f1787186742a9a884b73a43d8cf219f9b' or result == '91a73fd806ab2c005c13b4dc19130a884e909dea3f72d46e30266fe1a1f588d8', result
         finally:
             SCons.Node.Node.NodeInfo = SCons.Node.NodeInfoBase
 

SCons/Util.py

@@ -1669,11 +1669,58 @@ def AddMethod(obj, function, name=None):
 
 
 # Default hash function and format. SCons-internal.
-ALLOWED_HASH_FORMATS = ['md5', 'sha1', 'sha256']
+_DEFAULT_HASH_FORMATS = ['md5', 'sha1', 'sha256']
+ALLOWED_HASH_FORMATS = []
 _HASH_FUNCTION = None
 _HASH_FORMAT = None
 
 
+def set_allowed_viable_default_hashes():
+    """Checks if SCons has ability to call the default algorithms normally supported.
+
+    This util class is sometimes called prior to setting the user-selected hash algorithm,
+    meaning that on FIPS-compliant systems the library would default-initialize MD5
+    and throw an exception in set_hash_format. A common case is using the SConf options,
+    which can run prior to main, and thus ignore the options.hash_format variable.
+
+    This function checks the _DEFAULT_HASH_FORMATS and sets the ALLOWED_HASH_FORMATS
+    to only the ones that can be called.
+
+    Throws if no allowed hash formats are detected.
+    """
+    global ALLOWED_HASH_FORMATS
+    _last_error = None
+    # note: if you call this method repeatedly, example using timeout, this is needed.
+    # otherwise it keeps appending valid formats to the string
+    ALLOWED_HASH_FORMATS = []
+
+    for test_algorithm in _DEFAULT_HASH_FORMATS:
+        _test_hash = getattr(hashlib, test_algorithm, None)
+        # we know hashlib claims to support it... check to see if we can call it.
+        if _test_hash is not None:
+            # the hashing library will throw an exception on initialization in FIPS mode,
+            # meaning if we call the default algorithm returned with no parameters, it'll
+            # throw if it's a bad algorithm, otherwise it will append it to the known
+            # good formats.
+            try:
+                _test_hash()
+                ALLOWED_HASH_FORMATS.append(test_algorithm)
+            except ValueError as e:
+                _last_error = e
+                continue
+
+    if len(ALLOWED_HASH_FORMATS) == 0:
+        from SCons.Errors import SConsEnvironmentError  # pylint: disable=import-outside-toplevel
+        # chain the exception thrown with the most recent error from hashlib.
+        raise SConsEnvironmentError(
+            'No usable hash algorithms found.'
+            'Most recent error from hashlib attached in trace.'
+        ) from _last_error
+    return
+
+set_allowed_viable_default_hashes()
+
+
 def get_hash_format():
     """Retrieves the hash format or ``None`` if not overridden.
 
@@ -1702,22 +1749,54 @@ def set_hash_format(hash_format):
         if hash_format_lower not in ALLOWED_HASH_FORMATS:
             from SCons.Errors import UserError  # pylint: disable=import-outside-toplevel
 
-            raise UserError('Hash format "%s" is not supported by SCons. Only '
+            # user can select something not supported by their OS but normally supported by
+            # SCons, example, selecting MD5 in an OS with FIPS-mode turned on. Therefore we first
+            # check if SCons supports it, and then if their local OS supports it.
+            if hash_format_lower in _DEFAULT_HASH_FORMATS:
+                raise UserError('While hash format "%s" is supported by SCons, the '
+                                'local system indicates only the following hash '
+                                'formats are supported by the hashlib library: %s' %
+                                (hash_format_lower,
+                                ', '.join(ALLOWED_HASH_FORMATS))
+                )
+            # the hash format isn't supported by SCons in any case. Warn the user, and
+            # if we detect that SCons supports more algorithms than their local system
+            # supports, warn the user about that too.
+            else:
+                if ALLOWED_HASH_FORMATS == _DEFAULT_HASH_FORMATS:
+                    raise UserError('Hash format "%s" is not supported by SCons. Only '
                             'the following hash formats are supported: %s' %
                             (hash_format_lower,
-                             ', '.join(ALLOWED_HASH_FORMATS)))
+                             ', '.join(ALLOWED_HASH_FORMATS))
+                    )
+                else:
+                    raise UserError('Hash format "%s" is not supported by SCons. '
+                            'SCons supports more hash formats than your local system '
+                            'is reporting; SCons supports: %s. Your local system only '
+                            'supports: %s' %
+                            (hash_format_lower,
+                             ', '.join(_DEFAULT_HASH_FORMATS), 
+                             ', '.join(ALLOWED_HASH_FORMATS))
+                    )
 
+        # this is not expected to fail. If this fails it means the set_allowed_viable_default_hashes
+        # function did not throw, or when it threw, the exception was caught and ignored, or
+        # the global ALLOWED_HASH_FORMATS was changed by an external user.
         _HASH_FUNCTION = getattr(hashlib, hash_format_lower, None)
         if _HASH_FUNCTION is None:
             from SCons.Errors import UserError  # pylint: disable=import-outside-toplevel
 
             raise UserError(
-                'Hash format "%s" is not available in your Python interpreter.'
+                'Hash format "%s" is not available in your Python interpreter. '
+                'Expected to be supported algorithm by set_allowed_viable_default_hashes, '
+                'Assertion error in SCons.'
                 % hash_format_lower
             )
     else:
         # Set the default hash format based on what is available, defaulting
-        # to md5 for backwards compatibility.
+        # to the first supported hash algorithm (usually md5) for backwards compatibility.
+        # in FIPS-compliant systems this usually defaults to SHA1, unless that too has been
+        # disabled.
         for choice in ALLOWED_HASH_FORMATS:
             _HASH_FUNCTION = getattr(hashlib, choice, None)
             if _HASH_FUNCTION is not None:
@@ -1728,7 +1807,9 @@ def set_hash_format(hash_format):
 
             raise UserError(
                 'Your Python interpreter does not have MD5, SHA1, or SHA256. '
-                'SCons requires at least one.')
+                'SCons requires at least one. Expected to support one or more '
+                'during set_allowed_viable_default_hashes.'
+            )
 
 # Ensure that this is initialized in case either:
 #    1. This code is running in a unit test.

doc/man/scons.xml

@@ -1185,8 +1185,8 @@ to use the specified algorithm.</para>
 For example, <option>--hash-format=sha256</option> will create a SConsign
 database with name <filename>.sconsign_sha256.dblite</filename>.</para>
 
-<para>If this option is not specified, a hash format of
-md5 is used, and the SConsign database is
+<para>If this option is not specified, a the first supported hash format found
+is selected, and the SConsign database is
 <filename>.sconsign.dblite</filename>.</para>
 
 <para><emphasis>Available since &scons; 4.2.</emphasis></para>

test/sconsign/script/Configure.py

@@ -66,7 +66,7 @@
 
 test.run(arguments = '.')
 
-sig_re = r'[0-9a-fA-F]{32}'
+sig_re = r'[0-9a-fA-F]{32,64}'
 date_re = r'\S+ \S+ [ \d]\d \d\d:\d\d:\d\d \d\d\d\d'
 _sconf_temp_conftest_0_c = '.sconf_temp/conftest_%(sig_re)s_0.c'%locals()
 

test/sconsign/script/SConsignFile.py

@@ -141,7 +141,7 @@ def process(infp, outfp):
 
 test.run(arguments = '--implicit-cache .')
 
-sig_re = r'[0-9a-fA-F]{32}'
+sig_re = r'[0-9a-fA-F]{32,64}'
 
 test.run_sconsign(arguments = ".sconsign",
          stdout = r"""=== .:

test/sconsign/script/Signatures.py

@@ -149,7 +149,7 @@ def process(infp, outfp):
 
 test.run(arguments = '. --max-drift=1')
 
-sig_re = r'[0-9a-fA-F]{32}'
+sig_re = r'[0-9a-fA-F]{32,64}'
 date_re = r'\S+ \S+ [ \d]\d \d\d:\d\d:\d\d \d\d\d\d'
 
 test.run_sconsign(arguments = "-e hello.exe -e hello.obj sub1/.sconsign",

test/sconsign/script/dblite.py

@@ -109,7 +109,7 @@ def escape_drive_case(s):
 
 test.run(arguments = '. --max-drift=1')
 
-sig_re = r'[0-9a-fA-F]{32}'
+sig_re = r'[0-9a-fA-F]{32,64}'
 date_re = r'\S+ \S+ [ \d]\d \d\d:\d\d:\d\d \d\d\d\d'
 
 if sys.platform == 'win32':

test/sconsign/script/no-SConsignFile.py

@@ -152,7 +152,7 @@ def process(infp, outfp):
 
 test.run(arguments = '--implicit-cache --tree=prune .')
 
-sig_re = r'[0-9a-fA-F]{32}'
+sig_re = r'[0-9a-fA-F]{32,64}'
 
 expect = r"""hello.c: %(sig_re)s \d+ \d+
 hello.exe: %(sig_re)s \d+ \d+

testing/framework/TestSCons.py

@@ -1415,10 +1415,10 @@ def checkLogAndStdout(self, checks, results, cached,
                     for ext, flag in bld_desc:  # each file in TryBuild
                         if ext in ['.c', '.cpp']:
                             conf_filename = re.escape(os.path.join(sconf_dir, "conftest")) +\
-                                            r'_[a-z0-9]{32}_\d+%s' % re.escape(ext)
+                                            r'_[a-z0-9]{32,64}_\d+%s' % re.escape(ext)
                         elif ext == '':
                             conf_filename = re.escape(os.path.join(sconf_dir, "conftest")) +\
-                                            r'_[a-z0-9]{32}(_\d+_[a-z0-9]{32})?'
+                                            r'_[a-z0-9]{32,64}(_\d+_[a-z0-9]{32,64})?'
 
                         else:
                             # We allow the second hash group to be optional because
@@ -1430,7 +1430,7 @@ def checkLogAndStdout(self, checks, results, cached,
                             # TODO: perhaps revisit and/or fix file naming for intermediate files in
                             #  Configure context logic
                             conf_filename = re.escape(os.path.join(sconf_dir, "conftest")) +\
-                                            r'_[a-z0-9]{32}_\d+(_[a-z0-9]{32})?%s' % re.escape(ext)
+                                            r'_[a-z0-9]{32,64}_\d+(_[a-z0-9]{32,64})?%s' % re.escape(ext)
 
                         if flag == self.NCR:
                             # NCR = Non Cached Rebuild