Merge pull request #3805 from Flamefire/github_patch_detection

Correctly resolve templates for patches in extensions when uploading to GitHub

Author: boegel

easybuild/tools/github.py

@@ -34,6 +34,7 @@
 import getpass
 import glob
 import functools
+import itertools
 import os
 import random
 import re
@@ -1013,10 +1014,14 @@ def is_patch_for(patch_name, ec):
 
     patches = copy.copy(ec['patches'])
 
-    for ext in ec['exts_list']:
-        if isinstance(ext, (list, tuple)) and len(ext) == 3 and isinstance(ext[2], dict):
-            ext_options = ext[2]
-            patches.extend(ext_options.get('patches', []))
+    with ec.disable_templating():
+        # take into account both list of extensions (via exts_list) and components (cfr. Bundle easyblock)
+        for entry in itertools.chain(ec['exts_list'], ec.get('components', [])):
+            if isinstance(entry, (list, tuple)) and len(entry) == 3 and isinstance(entry[2], dict):
+                templates = {'name': entry[0], 'version': entry[1]}
+                options = entry[2]
+                patches.extend(p[0] % templates if isinstance(p, (tuple, list)) else p % templates
+                               for p in options.get('patches', []))
 
     for patch in patches:
         if isinstance(patch, (tuple, list)):

test/framework/github.py

@@ -33,12 +33,14 @@
 import random
 import re
 import sys
+import textwrap
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
 from time import gmtime
 from unittest import TextTestRunner
 
 import easybuild.tools.testing
 from easybuild.base.rest import RestClient
+from easybuild.framework.easyconfig.easyconfig import EasyConfig
 from easybuild.framework.easyconfig.tools import categorize_files_by_type
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, module_classes, update_build_option
@@ -803,53 +805,103 @@ def test_github_det_patch_specs(self):
         """Test for det_patch_specs function."""
 
         patch_paths = [os.path.join(self.test_prefix, p) for p in ['1.patch', '2.patch', '3.patch']]
-        file_info = {'ecs': [
-            {'name': 'A', 'patches': ['1.patch'], 'exts_list': []},
-            {'name': 'B', 'patches': [], 'exts_list': []},
-        ]
-        }
+        file_info = {'ecs': []}
+
+        rawtxt = textwrap.dedent("""
+            easyblock = 'ConfigureMake'
+            name = 'A'
+            version = '42'
+            homepage = 'http://foo.com/'
+            description = ''
+            toolchain = {"name":"GCC", "version": "4.6.3"}
+
+            patches = ['1.patch']
+        """)
+        file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt))
+        rawtxt = textwrap.dedent("""
+            easyblock = 'ConfigureMake'
+            name = 'B'
+            version = '42'
+            homepage = 'http://foo.com/'
+            description = ''
+            toolchain = {"name":"GCC", "version": "4.6.3"}
+        """)
+        file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt))
+
         error_pattern = "Failed to determine software name to which patch file .*/2.patch relates"
         self.mock_stdout(True)
         self.assertErrorRegex(EasyBuildError, error_pattern, gh.det_patch_specs, patch_paths, file_info, [])
         self.mock_stdout(False)
 
-        file_info['ecs'].append({'name': 'C', 'patches': [('3.patch', 'subdir'), '2.patch'], 'exts_list': []})
+        rawtxt = textwrap.dedent("""
+            easyblock = 'ConfigureMake'
+            name = 'C'
+            version = '42'
+            homepage = 'http://foo.com/'
+            description = ''
+            toolchain = {"name":"GCC", "version": "4.6.3"}
+
+            patches = [('3.patch', 'subdir'), '2.patch']
+        """)
+        file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt))
         self.mock_stdout(True)
         res = gh.det_patch_specs(patch_paths, file_info, [])
         self.mock_stdout(False)
 
-        self.assertEqual(len(res), 3)
-        self.assertEqual(os.path.basename(res[0][0]), '1.patch')
-        self.assertEqual(res[0][1], 'A')
-        self.assertEqual(os.path.basename(res[1][0]), '2.patch')
-        self.assertEqual(res[1][1], 'C')
-        self.assertEqual(os.path.basename(res[2][0]), '3.patch')
-        self.assertEqual(res[2][1], 'C')
+        self.assertEqual([i[0] for i in res], patch_paths)
+        self.assertEqual([i[1] for i in res], ['A', 'C', 'C'])
 
         # check if patches for extensions are found
-        file_info['ecs'][-1] = {
-            'name': 'patched_ext',
-            'patches': [],
-            'exts_list': [
+        rawtxt = textwrap.dedent("""
+            easyblock = 'ConfigureMake'
+            name = 'patched_ext'
+            version = '42'
+            homepage = 'http://foo.com/'
+            description = ''
+            toolchain = {"name":"GCC", "version": "4.6.3"}
+
+            exts_list = [
                 'foo',
                 ('bar', '1.2.3'),
                 ('patched', '4.5.6', {
-                    'patches': [('2.patch', 1), '3.patch'],
+                    'patches': [('%(name)s-2.patch', 1), '%(name)s-3.patch'],
                 }),
-            ],
-        }
+            ]
+        """)
+        patch_paths[1:3] = [os.path.join(self.test_prefix, p) for p in ['patched-2.patch', 'patched-3.patch']]
+        file_info['ecs'][-1] = EasyConfig(None, rawtxt=rawtxt)
+
+        self.mock_stdout(True)
+        res = gh.det_patch_specs(patch_paths, file_info, [])
+        self.mock_stdout(False)
+
+        self.assertEqual([i[0] for i in res], patch_paths)
+        self.assertEqual([i[1] for i in res], ['A', 'patched_ext', 'patched_ext'])
+
+        # check if patches for components are found
+        rawtxt = textwrap.dedent("""
+            easyblock = 'PythonBundle'
+            name = 'patched_bundle'
+            version = '42'
+            homepage = 'http://foo.com/'
+            description = ''
+            toolchain = {"name":"GCC", "version": "4.6.3"}
+
+            components = [
+                ('bar', '1.2.3'),
+                ('patched', '4.5.6', {
+                    'patches': [('%(name)s-2.patch', 1), '%(name)s-3.patch'],
+                }),
+            ]
+        """)
+        file_info['ecs'][-1] = EasyConfig(None, rawtxt=rawtxt)
 
         self.mock_stdout(True)
         res = gh.det_patch_specs(patch_paths, file_info, [])
         self.mock_stdout(False)
 
-        self.assertEqual(len(res), 3)
-        self.assertEqual(os.path.basename(res[0][0]), '1.patch')
-        self.assertEqual(res[0][1], 'A')
-        self.assertEqual(os.path.basename(res[1][0]), '2.patch')
-        self.assertEqual(res[1][1], 'patched_ext')
-        self.assertEqual(os.path.basename(res[2][0]), '3.patch')
-        self.assertEqual(res[2][1], 'patched_ext')
+        self.assertEqual([i[0] for i in res], patch_paths)
+        self.assertEqual([i[1] for i in res], ['A', 'patched_bundle', 'patched_bundle'])
 
     def test_github_restclient(self):
         """Test use of RestClient."""

test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py

@@ -28,6 +28,7 @@
 @author: Miguel Dias Costa (National University of Singapore)
 """
 from easybuild.framework.easyblock import EasyBlock
+from easybuild.framework.easyconfig import CUSTOM
 
 
 class PythonBundle(EasyBlock):
@@ -37,4 +38,7 @@ class PythonBundle(EasyBlock):
     def extra_options(extra_vars=None):
         if extra_vars is None:
             extra_vars = {}
+        extra_vars.update({
+            'components': [(), "List of components to install: tuples w/ name, version and easyblock to use", CUSTOM],
+        })
         return EasyBlock.extra_options(extra_vars)